workshop_rs/frontend/
parser.rs

1//! Native localized Workshop parser.
2//!
3//! Parses vanilla Workshop text directly into validated, locale-independent
4//! the canonical public [`Program`]. Localized actions, values, events, enums, and structural
5//! keywords resolve through the canonical catalog; malformed input,
6//! unknown spellings, and recognized-but-unsupported constructs are reported
7//! as distinct structured diagnostics with source spans.
8
9pub(crate) use std::collections::HashMap;
10use std::sync::OnceLock;
11
12pub(crate) use crate::core::signatures::ExpectedDomain;
13pub(crate) use crate::core::source::{Position, SourceFile, Span};
14use crate::program::Program;
15pub(crate) use crate::settings::table::KeyKind;
16pub(crate) use crate::settings::{PathPart, table};
17pub(crate) use crate::settings::{Settings, SettingsListElement, SettingsNode};
18pub(crate) use crate::wir::{
19    self, Action, Event, EventTarget, EventTeam, ModifyOp, PlayerEventKind, Value, ValueNode,
20};
21
22pub(crate) use super::lexer::{Token, TokenKind, tokenize};
23pub(crate) use crate::catalog::{Catalog, Kind, Locale};
24pub(crate) use crate::core::error::{Result, WorkshopError};
25
26/// Where action parsing stopped.
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub(crate) enum Stop {
29    /// The enclosing `}` was consumed.
30    SectionClosed,
31    /// The `End` keyword is next (not consumed).
32    End,
33    /// The `Else If` keyword is next (not consumed).
34    ElseIf,
35    /// The `Else` keyword is next (not consumed).
36    Else,
37}
38
39pub(crate) enum AssignmentOperator {
40    Set,
41    Modify(ModifyOp),
42}
43
44/// Parse localized Workshop text into Workshop IR using the catalog's
45/// canonical call-signature context. Ambiguous bare enum members resolve when
46/// their enclosing call pins one matching domain; unpinned ambiguity remains a
47/// structured canonical value. See [`parse_with_context`] when a
48/// consumer needs to provide additional signature context.
49pub fn parse(input: &str, catalog: &Catalog, locale: &Locale) -> Result<Program> {
50    parse_with_context(input, catalog, locale, catalog)
51}
52
53#[cfg(test)]
54pub(crate) fn parse_wir(input: &str, catalog: &Catalog, locale: &Locale) -> Result<wir::Program> {
55    parse_wir_with_context(input, catalog, locale, catalog)
56}
57
58/// Compatibility entry point for crate-internal storage tests and migration
59/// checks. Ordinary consumers should use [`parse`], which returns [`Program`].
60/// Parse localized Workshop text into Workshop IR, resolving ambiguous bare
61/// enum members from the enclosing call's canonical signature context (#111).
62///
63/// When a bare member spelling matches several enum domains, the parser asks
64/// [`ExpectedDomain::expected_domain`] for the domain the enclosing call's
65/// signature expects at that argument position; the member resolves only when
66/// that expected domain is one of the matching domains (i.e. the signature
67/// pins exactly one). Without a pin all matching candidates are preserved.
68pub fn parse_with_context(
69    input: &str,
70    catalog: &Catalog,
71    locale: &Locale,
72    context: &dyn ExpectedDomain,
73) -> Result<Program> {
74    parse_wir_with_context(input, catalog, locale, context).and_then(Program::from_wir)
75}
76
77pub(crate) fn parse_wir_with_context(
78    input: &str,
79    catalog: &Catalog,
80    locale: &Locale,
81    context: &dyn ExpectedDomain,
82) -> Result<wir::Program> {
83    let tokens = tokenize(input).map_err(|error| WorkshopError::Malformed {
84        message: error.message,
85        span: Some(synthetic_span(error.position)),
86    })?;
87    ParseContext {
88        tokens,
89        pos: 0,
90        source: input,
91        catalog,
92        locale: locale.clone(),
93        context,
94        expected_domain: None,
95        condition_pending: false,
96        in_condition: false,
97        call_stack: Vec::new(),
98        target: wir::Program::default(),
99        globals: HashMap::new(),
100        players: HashMap::new(),
101        subroutines: HashMap::new(),
102        next_global_index: 0,
103        next_player_index: 0,
104    }
105    .program()
106}
107
108/// A synthetic single-position span (used before a file registry exists).
109pub(crate) fn synthetic_span(position: Position) -> Span {
110    Span::new(crate::core::ids::Id::from_index(0), position, position)
111}
112
113pub(crate) struct ParseContext<'a> {
114    pub(crate) tokens: Vec<Token>,
115    pub(crate) pos: usize,
116    pub(crate) source: &'a str,
117    pub(crate) catalog: &'a Catalog,
118    pub(crate) locale: Locale,
119    /// Canonical signature context (#111): supplies the expected enum domain
120    /// for the call argument currently being parsed.
121    pub(crate) context: &'a dyn ExpectedDomain,
122    /// The expected enum domain for the value currently being parsed, set by
123    /// [`ParseContext::value_args`] from the enclosing call's signature.
124    pub(crate) expected_domain: Option<&'a str>,
125    /// Marks the next [`ParseContext::value`] call as a rule-level condition
126    /// (`conditions` entries, `If`/`Else If`/`While` conditions) so an action
127    /// call there is diagnosed as an action used as a condition (#340).
128    pub(crate) condition_pending: bool,
129    /// Whether the value currently being parsed is that top-level condition
130    /// expression; nested arguments always parse as ordinary values.
131    pub(crate) in_condition: bool,
132    pub(crate) call_stack: Vec<String>,
133    pub(crate) target: wir::Program,
134    pub(crate) globals: HashMap<String, wir::GlobalVarId>,
135    pub(crate) players: HashMap<String, wir::PlayerVarId>,
136    pub(crate) subroutines: HashMap<String, wir::SubroutineId>,
137    pub(crate) next_global_index: u32,
138    pub(crate) next_player_index: u32,
139}
140
141impl<'a> ParseContext<'a> {
142    /// Run `f` with the signature-pinned expected domain temporarily replaced,
143    /// restoring the enclosing domain afterwards (#111).
144    pub(crate) fn with_domain<T>(
145        &mut self,
146        domain: Option<&'a str>,
147        f: impl FnOnce(&mut Self) -> Result<T>,
148    ) -> Result<T> {
149        let saved = self.expected_domain;
150        self.expected_domain = domain;
151        let result = f(self);
152        self.expected_domain = saved;
153        result
154    }
155
156    /// Parse a value while `domain` is in effect (or `None` for value
157    /// positions a signature must not pin).
158    pub(crate) fn value_in_domain(&mut self, domain: Option<&'a str>) -> Result<wir::ValueId> {
159        self.with_domain(domain, |p| p.value())
160    }
161
162    pub(crate) fn resolve_entry(
163        &self,
164        kind: Kind,
165        spelling: &str,
166    ) -> Option<&'a crate::catalog::CatalogEntry> {
167        self.catalog
168            .resolve(kind, &self.locale, spelling)
169            .or_else(|| {
170                if self.locale != *self.catalog.primary_locale() {
171                    self.catalog
172                        .resolve(kind, self.catalog.primary_locale(), spelling)
173                } else {
174                    None
175                }
176            })
177    }
178
179    pub(crate) fn canonical_keyword(&self, spelling: &str) -> String {
180        self.catalog
181            .resolve(Kind::Structural, &self.locale, spelling)
182            .or_else(|| {
183                (self.locale != *self.catalog.primary_locale()).then(|| {
184                    self.catalog
185                        .resolve(Kind::Structural, self.catalog.primary_locale(), spelling)
186                })?
187            })
188            .map(|entry| entry.id.clone())
189            .unwrap_or_else(|| canonical_keyword(spelling).to_string())
190    }
191
192    pub(crate) fn line_has_assignment(&self) -> bool {
193        let tokens: Vec<_> = self.tokens[self.pos..]
194            .iter()
195            .take_while(|token| !matches!(token.kind, TokenKind::Semi | TokenKind::RBrace))
196            .collect();
197        tokens.iter().any(|token| {
198            matches!(&token.kind, TokenKind::Op(op) if matches!(op.as_str(), "=" | "+=" | "-=" | "*=" | "/=" | "%="))
199        }) || tokens.windows(2).any(|window| {
200            matches!(&window[0].kind, TokenKind::Word(_))
201                && matches!(&window[1].kind, TokenKind::Op(op) if op == "=")
202        })
203    }
204
205    /// Read the maximal phrase of consecutive words (space-joined). Phrases
206    /// may span lines because long Workshop action arguments wrap mid-phrase.
207    pub(crate) fn phrase(&mut self) -> Result<(String, Position, Position)> {
208        let mut words = Vec::new();
209        let (start, mut end) = match self.peek() {
210            Some(Token {
211                kind: TokenKind::Word(word),
212                start,
213                end,
214            }) => {
215                words.push(word.clone());
216                (start, end)
217            }
218            Some(Token {
219                kind: TokenKind::Number { text, .. },
220                start,
221                end,
222            }) => {
223                words.push(text.clone());
224                (start, end)
225            }
226            Some(token) => return Err(self.malformed("expected an identifier", &token)),
227            None => return Err(self.malformed("expected an identifier", self.eof())),
228        };
229        self.pos += 1;
230        while let Some(token) = self.peek() {
231            match token {
232                Token {
233                    kind: TokenKind::Word(word),
234                    end: word_end,
235                    ..
236                } => {
237                    if matches!(
238                        self.peek_at(1).map(|token| token.kind),
239                        Some(TokenKind::Op(equal)) if equal == "="
240                    ) {
241                        break;
242                    }
243                    words.push(word.clone());
244                    end = word_end;
245                    self.pos += 1;
246                }
247                // Enum members embed numbers (`Team 2`, `Ability 2`); the
248                // lexer splits them from the word, so phrases join Number
249                // tokens too (#119 reference evidence).
250                Token {
251                    kind: TokenKind::Number { text, .. },
252                    end: number_end,
253                    ..
254                } => {
255                    words.push(text.clone());
256                    end = number_end;
257                    self.pos += 1;
258                }
259                _ => break,
260            }
261        }
262        Ok((words.join(" "), start, end))
263    }
264
265    /// Read a single-line phrase (stops at a line boundary). Used for names
266    /// that are structurally one per line, such as variable declarations.
267    pub(crate) fn phrase_on_line(&mut self) -> Result<(String, Position, Position)> {
268        self.phrase_on_line_impl(false)
269    }
270
271    /// Read a settings list element name: adjacent tokens stay joined and
272    /// `:` is part of the name, preserving localized spellings such as
273    /// `66号公路` and `Watchpoint: Gibraltar`.
274    pub(crate) fn settings_list_name(&mut self) -> Result<(String, Position, Position)> {
275        self.phrase_on_line_impl(true)
276    }
277
278    fn phrase_on_line_impl(&mut self, list_name: bool) -> Result<(String, Position, Position)> {
279        let mut words = Vec::new();
280        let (start, mut end, line) = match self.peek() {
281            Some(Token {
282                kind: TokenKind::Word(word),
283                start,
284                end,
285            }) => {
286                words.push(word.clone());
287                (start, end, start.line)
288            }
289            Some(Token {
290                kind: TokenKind::Number { text, .. },
291                start,
292                end,
293            }) => {
294                words.push(text.clone());
295                (start, end, start.line)
296            }
297            Some(token) => return Err(self.malformed("expected an identifier", &token)),
298            None => return Err(self.malformed("expected an identifier", self.eof())),
299        };
300        self.pos += 1;
301        while let Some(token) = self.peek() {
302            let (word, word_start, word_end) = match token {
303                Token {
304                    kind: TokenKind::Word(word),
305                    start,
306                    end,
307                } => (word, start, end),
308                Token {
309                    kind: TokenKind::Number { text, .. },
310                    start,
311                    end,
312                } => (text, start, end),
313                Token {
314                    kind: TokenKind::Dot,
315                    start,
316                    end,
317                } => (".".to_string(), start, end),
318                Token {
319                    kind: TokenKind::Op(op),
320                    start,
321                    end,
322                } if matches!(op.as_str(), "-" | "%") => (op.clone(), start, end),
323                Token {
324                    kind: TokenKind::Colon,
325                    start,
326                    end,
327                } if list_name => (":".to_string(), start, end),
328                _ => break,
329            };
330            if word_start.line != line {
331                break;
332            }
333            if list_name && word_start == end {
334                words
335                    .last_mut()
336                    .expect("phrase has a first token")
337                    .push_str(&word);
338            } else {
339                words.push(word);
340            }
341            end = word_end;
342            self.pos += 1;
343        }
344        let joined = words
345            .join(" ")
346            .replace(" .", ".")
347            .replace(". ", ".")
348            .replace(" %", "%");
349        // Canonical spellings keep the spacing around ':' (e.g. fr-FR
350        // "Observatoire : Gibraltar"); list names must not collapse it.
351        let joined = if list_name {
352            joined
353        } else {
354            joined.replace(" : ", ":")
355        };
356        Ok((joined, start, end))
357    }
358
359    pub(crate) fn phrase_on_line_with_colon(&mut self) -> Result<(String, Position, Position)> {
360        let (mut phrase, start, mut end) = self.phrase_on_line()?;
361        if matches!(
362            self.peek(),
363            Some(Token {
364                kind: TokenKind::Colon,
365                ..
366            })
367        ) {
368            let colon_pos = self.pos;
369            self.next();
370            let (rest, _, rest_end) = self.phrase_on_line()?;
371            if matches!(
372                self.peek(),
373                Some(Token {
374                    kind: TokenKind::LBrace,
375                    ..
376                })
377            ) {
378                phrase.push(':');
379                phrase.push(' ');
380                phrase.push_str(&rest);
381                end = rest_end;
382            } else {
383                self.pos = colon_pos;
384            }
385        }
386        Ok((phrase, start, end))
387    }
388
389    pub(crate) fn enum_member_phrase(&mut self) -> Result<(String, Position, Position)> {
390        let first = self
391            .peek()
392            .ok_or_else(|| self.malformed("expected an enum member", self.eof()))?;
393        let start = first.start;
394        let line = first.start.line;
395        let mut end = first.end;
396        let mut parts = Vec::new();
397        while let Some(token) = self.peek() {
398            if token.start.line != line
399                || matches!(token.kind, TokenKind::RParen | TokenKind::Comma)
400            {
401                break;
402            }
403            self.pos += 1;
404            end = token.end;
405            parts.push(raw_token_text(&token.kind));
406        }
407        if parts.is_empty() {
408            return Err(self.malformed("expected an enum member", &first));
409        }
410        Ok((
411            parts
412                .join(" ")
413                .replace(" : ", ":")
414                .replace(" .", ".")
415                .replace(". ", "."),
416            start,
417            end,
418        ))
419    }
420
421    /// Read a text line (tokens until `;`), joining words and dashes into
422    /// the literal text, and consume the terminating `;`. Returns the text
423    /// and the span covering the line's content tokens.
424    pub(crate) fn line_text(&mut self) -> Result<(String, Option<Span>)> {
425        let mut parts = Vec::new();
426        let mut start = None;
427        let mut end = None;
428        loop {
429            match self.peek() {
430                Some(Token {
431                    kind: TokenKind::Semi,
432                    ..
433                }) => {
434                    self.pos += 1;
435                    break;
436                }
437                Some(token) => {
438                    start.get_or_insert(token.start);
439                    end = Some(token.end);
440                    match &token.kind {
441                        TokenKind::Word(word) => parts.push(word.clone()),
442                        TokenKind::Op(op) if op == "-" => parts.push("-".to_string()),
443                        TokenKind::Number { value, .. } => parts.push(value.to_string()),
444                        TokenKind::Dot => parts.push(".".to_string()),
445                        TokenKind::Colon => parts.push(":".to_string()),
446                        _ => return Err(self.malformed("expected a text line", &token)),
447                    }
448                    self.pos += 1;
449                }
450                None => {
451                    return Err(self.malformed("unexpected end of input in line", self.eof()));
452                }
453            }
454        }
455        Ok((
456            parts
457                .join(" ")
458                .replace(" .", ".")
459                .replace(". ", ".")
460                .replace(" : ", ":"),
461            start
462                .zip(end)
463                .map(|(start, end)| Span::new(self.file(), start, end)),
464        ))
465    }
466
467    /// Consume a known keyword phrase, verifying its spelling.
468    pub(crate) fn consume_phrase(&mut self, expected: &str) -> Result<()> {
469        let (phrase, _, _) = self.phrase()?;
470        if phrase != expected {
471            return Err(self.malformed(&format!("expected '{expected}'"), self.previous()));
472        }
473        Ok(())
474    }
475
476    pub(crate) fn expect_keyword(&mut self, expected: &str) -> Result<Position> {
477        match self.next() {
478            Some(Token {
479                kind: TokenKind::Word(word),
480                start,
481                ..
482            }) if self.canonical_keyword(&word) == expected => Ok(start),
483            Some(token) => Err(self.malformed(&format!("expected '{expected}'"), &token)),
484            None => Err(self.malformed(&format!("expected '{expected}'"), self.eof())),
485        }
486    }
487
488    pub(crate) fn expect(&mut self, kind: TokenKind, message: &str) -> Result<()> {
489        match self.next() {
490            Some(token) if token.kind == kind => Ok(()),
491            Some(token) => Err(self.malformed(message, &token)),
492            None => Err(self.malformed(message, self.eof())),
493        }
494    }
495
496    /// Expect the `;` terminating a statement. When the token found instead
497    /// sits on a later line, the missing `;` is reported at the end of the
498    /// statement it should have closed rather than at that unrelated token
499    /// (#340); a same-line offender keeps pointing at the token itself.
500    pub(crate) fn expect_semi(&mut self, message: &str) -> Result<()> {
501        match self.peek() {
502            Some(Token {
503                kind: TokenKind::Semi,
504                ..
505            }) => {
506                self.pos += 1;
507                Ok(())
508            }
509            Some(token) if token.start.line > self.previous().end.line => {
510                let end = self.previous().end;
511                Err(WorkshopError::Malformed {
512                    message: message.to_string(),
513                    span: Some(Span::new(self.file(), end, end)),
514                })
515            }
516            Some(token) => Err(self.malformed(message, &token)),
517            None => Err(self.malformed(message, self.eof())),
518        }
519    }
520
521    pub(crate) fn expect_string(&mut self, message: &str) -> Result<String> {
522        match self.next() {
523            Some(Token {
524                kind: TokenKind::String(content),
525                ..
526            }) => Ok(content),
527            Some(token) => Err(self.malformed(message, &token)),
528            None => Err(self.malformed(message, self.eof())),
529        }
530    }
531
532    pub(crate) fn malformed(&self, message: &str, token: &Token) -> WorkshopError {
533        WorkshopError::Malformed {
534            message: message.to_string(),
535            span: Some(Span::new(self.file(), token.start, token.end)),
536        }
537    }
538
539    pub(crate) fn unknown(&self, kind: &'static str, spelling: &str) -> WorkshopError {
540        WorkshopError::Unknown {
541            kind,
542            spelling: spelling.to_string(),
543            locale: self.locale.clone(),
544            span: None,
545        }
546    }
547
548    pub(crate) fn peek(&self) -> Option<Token> {
549        self.tokens.get(self.pos).cloned()
550    }
551
552    pub(crate) fn peek_at(&self, offset: usize) -> Option<Token> {
553        self.tokens.get(self.pos + offset).cloned()
554    }
555
556    pub(crate) fn next(&mut self) -> Option<Token> {
557        let token = self.tokens.get(self.pos).cloned();
558        if token.is_some() {
559            self.pos += 1;
560        }
561        token
562    }
563
564    pub(crate) fn previous(&self) -> &Token {
565        self.tokens
566            .get(self.pos.saturating_sub(1))
567            .unwrap_or_else(|| self.tokens.last().unwrap())
568    }
569
570    pub(crate) fn previous_span(&self) -> (Position, Position) {
571        let token = self.previous();
572        (token.start, token.end)
573    }
574
575    pub(crate) fn span_here(&self) -> (Position, Position) {
576        let token = self
577            .peek()
578            .unwrap_or_else(|| self.tokens.last().unwrap().clone());
579        (token.start, token.end)
580    }
581
582    pub(crate) fn eof(&self) -> &Token {
583        self.tokens.last().unwrap()
584    }
585
586    pub(crate) fn file(&self) -> crate::core::ids::Id<SourceFile> {
587        crate::core::ids::Id::from_index(0)
588    }
589}
590
591pub(crate) fn canonical_keyword(keyword: &str) -> &str {
592    static KEYWORDS: OnceLock<HashMap<String, String>> = OnceLock::new();
593    KEYWORDS
594        .get_or_init(|| {
595            serde_json::from_str(include_str!("structural_keywords.json"))
596                .expect("structural keyword data is valid JSON")
597        })
598        .get(keyword)
599        .map(String::as_str)
600        .unwrap_or(keyword)
601}
602
603pub(crate) fn raw_token_text(kind: &TokenKind) -> String {
604    match kind {
605        TokenKind::Word(value) => value.clone(),
606        TokenKind::Number { text, .. } => text.clone(),
607        TokenKind::String(value) => format!("\"{}\"", value.replace('"', "\\\"")),
608        TokenKind::Op(value) => value.clone(),
609        TokenKind::LParen => "(".to_string(),
610        TokenKind::RParen => ")".to_string(),
611        TokenKind::Comma => ",".to_string(),
612        TokenKind::Semi => ";".to_string(),
613        TokenKind::LBrace => "{".to_string(),
614        TokenKind::RBrace => "}".to_string(),
615        TokenKind::Colon => ":".to_string(),
616        TokenKind::Dot => ".".to_string(),
617        TokenKind::LBracket => "[".to_string(),
618        TokenKind::RBracket => "]".to_string(),
619        TokenKind::Eof => String::new(),
620    }
621}
workshop_rs/frontend/parser.rs

workshop_rs/frontend/
parser.rs
