rbx_rsml/parser/
mod.rs

1use crate::{macros::{MacroGroup, MacroTokenIterator}, string_clip::StringClip};
2use crate::lexer::Token;
3use guarded::guarded_unwrap;
4use indexmap::IndexSet;
5use num_traits::Num;
6use palette::Srgb;
7use tree_node_group::{AnyTreeNode, AnyTreeNodeMut, TreeNodeType};
8use std::{fmt::Debug, ops::Deref, str::FromStr, sync::LazyLock};
9use phf_macros::phf_map;
10use rbx_types::{Color3uint8, Content, EnumItem, UDim, Variant};
11use regex::Regex;
12
13mod tree_node_group;
14pub use tree_node_group::{TreeNodeGroup, TreeNode};
15
16mod tuple;
17use tuple::Tuple;
18
19mod selector;
20use selector::Selector;
21
22mod datatype_group;
23use datatype_group::DatatypeGroup;
24pub use datatype_group::Datatype;
25
26mod variants;
27pub use variants::EnumItemFromNameAndValueName;
28
29mod operator;
30use operator::Operator;
31
32mod colors {
33    include!(concat!(env!("OUT_DIR"), "/colors.rs"));
34}
35use colors::{TAILWIND_COLORS, BRICK_COLORS, CSS_COLORS, SKIN_COLORS};
36
37const MULTI_LINE_STRING_STRIP_LEFT_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[ \t\f]*\n+").unwrap());
38
39type TokenWithResult<'a, R> = (Option<Token>, R);
40
41fn parse_number_string<D>(num_str: &str) -> Result<D, <D as FromStr>::Err>
42where
43    D: Num + FromStr,
44    <D as FromStr>::Err: Debug 
45{   
46    let mut num_str = num_str.to_string();
47    num_str.retain(|c| !r#"_"#.contains(c));
48    num_str.parse::<D>()
49}
50
51struct Parser<'a> {
52    lexer: &'a mut logos::Lexer<'a, Token>,
53
54    /// The names of the macros that the parser is currently inside of.
55    /// Used to prevent infinite recursion.
56    current_macros: IndexSet<(String, usize)>,
57
58    macros: &'a MacroGroup,
59    injected_macro_tokens: Vec<MacroTokenIterator<'a>>,
60
61    tree_nodes: TreeNodeGroup,
62    current_tree_node_idx: TreeNodeType,
63
64    tuples: Vec<Tuple>,
65
66    did_advance: bool,
67}
68
69impl<'a> Parser<'a> {
70    fn new(lexer: &'a mut logos::Lexer<'a, Token>, macros: &'a MacroGroup) -> Self {
71        Self {
72            lexer,
73
74            current_macros: IndexSet::new(),
75            macros,
76
77            injected_macro_tokens: vec![],
78
79            tree_nodes: TreeNodeGroup::new(),
80            current_tree_node_idx: TreeNodeType::Root,
81
82            tuples: vec![],
83
84            did_advance: false,
85        }
86    }
87
88    fn inject_tokens(&mut self, injected_macro_tokens: MacroTokenIterator<'a>) {
89        self.injected_macro_tokens.push(injected_macro_tokens);
90    }
91
92    fn core_next(&mut self) -> Option<Result<Token, ()>> {
93        if let Some(last_injected_macro_tokens) = self.injected_macro_tokens.last_mut() {
94            let next_injected = last_injected_macro_tokens.next();
95
96            if next_injected.is_some() {
97                next_injected
98
99            } else {
100                self.injected_macro_tokens.pop();
101                self.current_macros.pop();
102                self.core_next()
103            }
104
105        } else {
106            self.lexer.next()
107        }
108    }
109
110    fn slice(&self) -> &'a str {
111        if let Some(injected_macro_tokens) = self.injected_macro_tokens.last() {
112            injected_macro_tokens.slice()
113        } else {
114            self.lexer.slice()
115        }
116    }
117
118    // The `advance` method performs work which would be redundant for:
119    // `parse_comment_multi`, `parse_comment_single`, `parse_string_multi_end`.
120    // So this core method serves to strip all of it away.
121    fn core_advance(&mut self) -> Option<Token> {
122        self.did_advance = true;
123
124        loop {
125            match self.core_next() {
126                Some(Ok(token)) => break Some(token),
127                None => return None,
128                _ => ()
129            }
130        }
131    }
132
133    fn advance(self: &mut Parser<'a>) -> Option<Token> {
134        let token = guarded_unwrap!(self.core_advance(), return None);
135
136        let token = parse_comment_multi(self, token).unwrap_or(token);
137
138        let token = parse_comment_single(self, token).unwrap_or(token);
139
140        parse_macro_call(self, token)
141    }
142
143    // create remove tuple for when a tuple is coerced into a datatype and thusly
144    // never used again.
145    fn push_tuple(self: &mut Parser<'a>, tuple: Tuple) -> usize {
146        let tuples = &mut self.tuples;
147
148        let new_tuple_idx = tuples.len();
149        tuples.push(tuple);
150
151        new_tuple_idx
152    }
153
154    fn get_tuple(self: &mut Parser<'a>, tuple_idx: usize) -> Option<&Tuple> {
155        self.tuples.get(tuple_idx)
156    }
157
158    fn get_tuple_mut<'b>(self: &'b mut Parser<'a>, tuple_idx: usize) -> Option<&'b mut Tuple> {
159        self.tuples.get_mut(tuple_idx)
160    }
161}
162
163fn parse_macro_call<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
164    if !matches!(token, Token::MacroCall) { return Some(token) }
165
166    let macro_name_with_suffix = parser.slice();
167    let macro_name: String = macro_name_with_suffix.chars().take(macro_name_with_suffix.chars().count() - 1).collect();
168
169    let next_token = parser.advance();
170    if !matches!(next_token, Some(Token::ParensOpen)) { return next_token }
171
172    let mut scope_nestedness = 0usize;
173    let mut parens_nestedness = 0usize;
174    let mut parens_nestedness_is_zero = true;
175
176    let mut args_tokens: Vec<Vec<(Token, &'a str)>> = vec![];
177    let mut current_args_tokens: Vec<(Token, &'a str)> = vec![];
178    
179    let mut next_token = parser.advance()?;
180    loop {
181        match next_token {
182            Token::ScopeOpen => scope_nestedness += 1,
183            Token::ScopeClose => scope_nestedness -= 1,
184
185            Token::ParensOpen => {
186                parens_nestedness += 1;
187                parens_nestedness_is_zero = false;
188            },
189            Token::ParensClose => {
190                if parens_nestedness_is_zero {
191                    if current_args_tokens.len() != 0 {
192                        args_tokens.push(current_args_tokens);
193                    }
194                    break
195
196                } else {
197                    parens_nestedness -= 1;
198                    parens_nestedness_is_zero = parens_nestedness == 0;
199                }
200            }
201
202            Token::Comma => {
203                if scope_nestedness == 0 && parens_nestedness_is_zero && current_args_tokens.len() != 0 {
204                    args_tokens.push(current_args_tokens);
205                    current_args_tokens = vec![];
206
207                    next_token = parser.advance()?;
208                    continue;
209                }
210            }
211
212            _ => ()
213        }
214        
215        current_args_tokens.push((next_token, parser.slice()));
216        next_token = parser.advance()?;
217    }
218
219    let args_len = args_tokens.len();
220
221    let macro_data = guarded_unwrap!(parser.macros.get(&macro_name, args_len), return parser.advance());
222
223    let current_macro = (macro_name, args_len);
224
225    // Prevents infinite recursion.
226    if parser.current_macros.contains(&current_macro) {
227        return parser.advance()
228    }
229
230    parser.inject_tokens(macro_data.iter(Some(args_tokens)));
231
232    parser.current_macros.insert(current_macro);
233
234    parser.advance()
235}
236
237fn parse_comment_multi_end<'a>(
238    parser: &mut Parser<'a>, start_equals_amount: usize
239) -> Option<Token> {
240    // We keep advancing tokens until we find a closing multiline string
241    // token with the same amount of equals signs as the start token.
242    loop {
243        let token = parser.core_advance()?;
244
245        if let Token::StringMultiEnd = token {
246            let end_token_value = parser.slice();
247            let end_equals_amount = end_token_value.clip(1, 1).len();
248
249            if start_equals_amount == end_equals_amount {
250                return parser.core_advance()
251            }
252        }
253    }
254}
255
256fn parse_comment_multi<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
257    if let Token::CommentMultiStart = token {
258        let token_value = parser.slice();
259        let start_equals_amount = token_value.clip(3, 1).len();
260
261        return parse_comment_multi_end(parser, start_equals_amount);
262    };
263
264    None
265}
266
267fn parse_comment_single<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
268    if !matches!(token, Token::CommentSingle) { return None }
269
270    parser.core_advance()
271}
272
273fn parse_scope_open<'a>(parser: &mut Parser<'a>, token: Token, selector: Option<String>) -> Option<Token> {
274    if !matches!(token, Token::ScopeOpen) { return Some(token) }
275 
276    let new_tree_node_idx = parser.tree_nodes.nodes_len();
277    let new_tree_node_idx_as_parent = TreeNodeType::Node(new_tree_node_idx);
278
279    let previous_tree_node = parser.tree_nodes.get_node_mut(parser.current_tree_node_idx);
280    match previous_tree_node {
281        AnyTreeNodeMut::Root(node) => node.unwrap().child_rules.push(new_tree_node_idx),
282        AnyTreeNodeMut::Node(node) => node.unwrap().child_rules.push(new_tree_node_idx)
283    }
284
285    let current_tree_node = TreeNode::new(parser.current_tree_node_idx, selector);
286    parser.tree_nodes.add_node(current_tree_node);
287
288    parser.current_tree_node_idx = new_tree_node_idx_as_parent;
289
290    parser.advance()
291}
292
293fn parse_scope_delimiter<'a>(
294    parser: &mut Parser<'a>,
295    token: Token,
296    delim_token: Option<Token>
297) -> TokenWithResult<'a, Option<Token>> {
298    if matches!(token, Token::Comma) {
299        let next_token = guarded_unwrap!(parser.advance(), return (None, delim_token));
300        return parse_scope_delimiter(parser, next_token, Some(token))
301
302    } else {
303        return (Some(token), delim_token)
304    }
305}
306
307fn parse_scope_selector_start<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
308    // The `Token::Text` case is handled in `parse_property`.
309    if !matches!(token, 
310        Token::NameIdentifier | Token::PsuedoIdentifier | Token::StateOrEnumIdentifier |
311        Token::TagOrEnumIdentifier | Token::ScopeToDescendants | Token::ScopeToChildren
312    ) { return Some(token) }
313
314    let selector = Selector::new(parser.slice(), token);
315
316    let token = parser.advance()?;
317    return parse_scope_selector(parser, token, selector);
318}
319
320fn parse_scope_selector<'a>(
321    parser: &mut Parser<'a>, mut token: Token, mut selector: Selector
322) -> Option<Token> {
323    loop {
324        // Advances the parser until no delimiter token is found.
325        let parsed_delimiter = parse_scope_delimiter(parser, token, None);
326        token = guarded_unwrap!(parsed_delimiter.0, return None);
327
328        if matches!(token, 
329            Token::NameIdentifier | Token::PsuedoIdentifier | Token::StateOrEnumIdentifier |
330            Token::TagOrEnumIdentifier | Token::ScopeToDescendants | Token::ScopeToChildren |
331            Token::Text
332        ) {
333            // Appending the comma token here ensures selectors can't end with delimiters.
334            if let Some(delim_token) = parsed_delimiter.1 {
335                selector.append(",", delim_token);
336            }
337            
338            selector.append(parser.slice(), token);
339
340            token = parser.advance()?;
341
342        } else { break }
343    };
344
345    return parse_scope_open(parser, token, Some(selector.content))
346}
347
348
349fn parse_scope_close<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
350    if !matches!(token, Token::ScopeClose) { return Some(token) }
351
352    let current_tree_node = parser.tree_nodes.get(parser.current_tree_node_idx);
353    match current_tree_node {
354        AnyTreeNode::Node(node) => parser.current_tree_node_idx = node.unwrap().parent,
355        _ => ()
356    }
357
358    return parser.advance()
359}
360
361fn parse_string_multi_end<'a>(
362    parser: &mut Parser<'a>, start_equals_amount: usize
363) -> TokenWithResult<'a, String> {
364    let mut string_data = String::new();
365
366    // We keep advancing tokens until we find a closing multiline string
367    // token with the same amount of equals signs as the start token.
368    loop {
369        match parser.lexer.next() {
370            Some(Ok(token)) => {
371                if let Token::StringMultiEnd = token {
372                    let end_token_value = parser.slice();
373                    let end_equals_amount = end_token_value.clip(1, 1).len();
374        
375                    if start_equals_amount == end_equals_amount {
376                        return (parser.core_advance(), string_data)
377                    }
378                }
379
380                string_data += parser.slice();
381            },
382
383            Some(Err(_)) => {
384                string_data += parser.slice();
385            },
386
387            None => return (None, string_data)
388        };
389    }
390}
391
392fn parse_string_multi_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
393    if let Token::StringMultiStart = token {
394        let token_value = parser.slice();
395        let start_equals_amount = token_value.clip(1, 1).len();
396
397        let (token, string_data) = parse_string_multi_end(parser, start_equals_amount);
398
399        // Luau strips multiline strings up until the first occurance of a newline character.
400        // So we will mimic this behaviour.
401        let string_data = MULTI_LINE_STRING_STRIP_LEFT_REGEX.replace(&string_data, "").to_string();
402        return (token, Some(Datatype::Variant(Variant::String(string_data))))
403    };
404
405    (Some(token), None)
406}
407
408fn parse_content_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
409    if let Token::RobloxContent = token {
410        let content = parser.slice();
411        return (parser.advance(), Some(Datatype::Variant(Variant::Content(Content::from(content)))))
412    }
413
414    (Some(token), None)
415}
416
417fn parse_string_single_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
418    match token {
419        Token::StringSingle => {
420            let str = parser.slice();
421            let datatype = Some(Datatype::Variant(Variant::String(str.clip(1, 1).to_string())));
422            return (parser.advance(), datatype)
423        },
424
425        Token::RobloxAsset => {
426            let str = parser.slice();
427            let datatype = Some(Datatype::Variant(Variant::String(str.to_string())));
428            return (parser.advance(), datatype)
429        },
430
431        _ => (Some(token), None)
432    }
433}
434
435fn parse_string_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
436    let parsed = parse_string_single_datatype(parser, token);
437    if parsed.1.is_some() { return parsed }
438
439    let parsed = parse_string_multi_datatype(parser, token);
440    if parsed.1.is_some() { return parsed }
441
442    (Some(token), None)
443}
444
445fn parse_number_offset<'a>(parser: &mut Parser<'a>, token: Token, num_str: &str) -> TokenWithResult<'a, Option<Datatype>> {
446    if !matches!(token, Token::Offset) { return (Some(token), None) }
447
448    let datatype = Some(Datatype::Variant(Variant::UDim(UDim::new(
449        0.0, 
450        match parse_number_string::<i32>(num_str) {
451            Ok(int32) => int32,
452            Err(_) => parse_number_string::<f32>(num_str).unwrap() as i32
453        }
454    ))));
455
456    return (parser.advance(), datatype)
457}
458
459fn parse_number_scale<'a>(parser: &mut Parser<'a>, token: Token, num_str: &str) -> TokenWithResult<'a, Option<Datatype>> {
460    if !matches!(token, Token::ScaleOrOpMod) { return (Some(token), None) }
461
462    let datatype = Some(Datatype::Variant(Variant::UDim(UDim::new(parse_number_string::<f32>(num_str).unwrap() / 100.0, 0))));
463
464    return (parser.advance(), datatype)
465}
466
467fn parse_number_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
468    if let Token::Number = token {
469        let token_value = parser.slice();
470        let token = parser.advance();
471
472        if let Some(token) = token {
473            let parsed = parse_number_offset(parser, token, token_value);
474            if parsed.1.is_some() { return parsed }
475
476            let parsed = parse_number_scale(parser, token, token_value);
477            if parsed.1.is_some() { return parsed }
478        }
479
480        let datatype = Some(Datatype::Variant(Variant::Float32(parse_number_string::<f32>(token_value).unwrap())));
481
482        return (token, datatype)
483    }
484
485    (Some(token), None)
486}
487
488fn parse_operator_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
489    match Operator::from_token(token) {
490        Some(operator) => (parser.advance(), Some(Datatype::Operator(operator.clone()))),
491        None => (Some(token), None)
492    }
493}
494
495fn parse_enum_tokens<'a>(parser: &mut Parser<'a>, token_history: &mut Vec<Token>) -> Option<Datatype> {
496    let token_history_len = token_history.len();
497    let mut first_stop_idx = 0;
498
499    let mut enum_name = None;
500    for idx in 0..=token_history_len {
501        let token = &token_history[idx];
502        
503        if let Token::Text = token {
504            let text = parser.slice();
505            enum_name = Some(text);
506            
507            first_stop_idx = idx;
508            break
509        }
510    };
511    let enum_name = guarded_unwrap!(enum_name, return None);
512
513    let mut enum_value_name = None;
514    for idx in (first_stop_idx + 1)..=token_history_len {
515        let token = &token_history[idx];
516
517        if let Token::Text = token {
518            let text = parser.slice();
519            enum_value_name = Some(text);
520            break
521        }
522    };
523    let enum_value_name = guarded_unwrap!(enum_value_name, return None);
524
525    let enum_item = guarded_unwrap!(EnumItem::from_name_and_value_name(enum_name, enum_value_name), return Some(Datatype::None));
526    return Some(Datatype::Variant(Variant::EnumItem(enum_item)))
527}
528
529fn parse_full_enum<'a>(
530    parser: &mut Parser<'a>, token: Token, token_history: &mut Vec<Token>
531) -> TokenWithResult<'a, Option<Datatype>> {
532    if matches!(token, 
533        Token::TagOrEnumIdentifier | Token::StateOrEnumIdentifier | Token::Text
534    ) {
535        let token = parser.advance();
536
537        if let Some(token) = token {
538            token_history.push(token);
539
540            let parsed = parse_full_enum(parser, token, token_history);
541            if parsed.1.is_some() { return parsed }
542        }
543    }
544
545    if token_history.len() == 0 { return (Some(token), None) }
546
547    return (Some(token), parse_enum_tokens(parser, token_history))
548}
549
550
551fn parse_enum_keyword<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
552    if !matches!(token, Token::EnumKeyword) { return (Some(token), None) }
553
554    let token = guarded_unwrap!(parser.advance(), return (None, None) );
555
556    let mut token_history = vec![];
557    parse_full_enum(parser, token, &mut token_history)
558}
559
560const SHORTHAND_REBINDS: phf::Map<&'static str, &'static str> = phf_map! {
561    "FlexMode" => "UIFlexMode",
562    "HorizontalFlex" => "UIFlexAlignment",
563    "VerticalFlex" => "UIFlexAlignment"
564};
565
566fn parse_enum_shorthand<'a>(parser: &mut Parser<'a>, token: Token, key: Option<&str>) -> TokenWithResult<'a, Option<Datatype>> {
567    if !matches!(token, Token::StateOrEnumIdentifier) { return (Some(token), None) }
568
569    let token = guarded_unwrap!(parser.advance(), return (None, None));
570
571    let enum_value_name = if let Token::Text = token { parser.slice() } else { return (Some(token), None) };
572
573    if let Some(key) = key {
574        let enum_name = SHORTHAND_REBINDS.get(key).unwrap_or(&key);
575
576        let enum_item = guarded_unwrap!(
577            EnumItem::from_name_and_value_name(enum_name, enum_value_name), return (parser.advance(), Some(Datatype::None))
578        );
579        return (parser.advance(), Some(Datatype::Variant(Variant::EnumItem(enum_item))))
580
581    } else {
582        let datatype = Datatype::IncompleteEnumShorthand(enum_value_name.into());
583        return (parser.advance(), Some(datatype))
584    }
585
586    
587}
588
589fn parse_enum_datatype<'a>(parser: &mut Parser<'a>, token: Token, key: Option<&str>) -> TokenWithResult<'a, Option<Datatype>> {
590    let parsed = parse_enum_keyword(parser, token);
591    if parsed.1.is_some() { return parsed }
592
593    let parsed = parse_enum_shorthand(parser, token, key);
594    if parsed.1.is_some() { return parsed }
595
596    (Some(token), None)
597}
598
599fn parse_nil_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
600    if !matches!(token, Token::Nil) { return (Some(token), None) }
601
602    (parser.advance(), Some(Datatype::None))
603}
604
605fn parse_boolean_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
606    match token {
607        Token::BoolTrue => {
608            (parser.advance(), Some(Datatype::Variant(Variant::Bool(true))))
609        },
610        Token::BoolFalse => {
611            (parser.advance(), Some(Datatype::Variant(Variant::Bool(false))))
612        },
613        _ => (Some(token), None)
614    }
615}
616
617#[allow(warnings)]
618fn parse_preset_tailwind_color<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
619    if let Token::ColorTailwind = token {
620        let color_name = parser.slice();
621        let datatype = TAILWIND_COLORS.get(&color_name.to_lowercase())
622            .and_then(|color| Some(Datatype::Oklab(**color.deref())));
623
624        return (parser.advance(), datatype)
625    }
626
627    (Some(token), None)
628}
629
630#[allow(warnings)]
631fn parse_preset_skin_color<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
632    if let Token::ColorSkin = token {
633        let color_name = parser.slice();
634        let datatype = SKIN_COLORS.get(&color_name.to_lowercase())
635            .and_then(|color| Some(Datatype::Oklab(**color.deref())));
636
637        return (parser.advance(), datatype)
638    }
639
640    (Some(token), None)
641}
642
643#[allow(warnings)]
644fn parse_preset_css_color<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
645    if let Token::ColorCss = token {
646        let color_name = parser.slice();
647        let datatype = CSS_COLORS.get(&color_name.to_lowercase())
648            .and_then(|color| Some(Datatype::Oklab(**color.deref())));
649
650        return (parser.advance(), datatype)
651    }
652
653    (Some(token), None)
654}
655
656#[allow(warnings)]
657fn parse_preset_brick_color<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
658    if let Token::ColorBrick = token {
659        let color_name = parser.slice();
660        let datatype = BRICK_COLORS.get(&color_name.to_lowercase())
661            .and_then(|color| Some(Datatype::Oklab(**color.deref())));
662
663        return (parser.advance(), datatype)
664    }
665
666    (Some(token), None)
667}
668
669fn parse_preset_color_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
670    let parsed = parse_preset_tailwind_color(parser, token);
671    if parsed.1.is_some() { return parsed }
672
673    let parsed = parse_preset_skin_color(parser, token);
674    if parsed.1.is_some() { return parsed }
675
676    let parsed = parse_preset_css_color(parser, token);
677    if parsed.1.is_some() { return parsed }
678
679    let parsed = parse_preset_brick_color(parser, token);
680    if parsed.1.is_some() { return parsed }
681
682    return (Some(token), None)
683}
684
685fn normalize_hex(hex: &str) -> String {
686    let hex = hex.trim_start_matches('#');
687
688    match hex.len() {
689        3 | 6 => hex.into(),
690        1..=5 => format!("{:0<6}", hex),
691        _ => hex.into(),
692    }
693}
694
695fn parse_hex_color_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
696    if let Token::ColorHex = token {
697        let hex = parser.slice();
698
699        let color: Srgb<u8> = normalize_hex(hex).parse().unwrap();
700        let datatype = Datatype::Variant(Variant::Color3uint8(Color3uint8::new(
701            color.red,
702            color.green,
703            color.blue,
704        )));
705
706        return (parser.advance(), Some(datatype))
707    }
708
709    return (None, None)
710}
711
712fn parse_color_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
713    let parsed = parse_preset_color_datatype(parser, token);
714    if parsed.1.is_some() { return parsed }
715
716    let parsed = parse_hex_color_datatype(parser, token);
717    if parsed.1.is_some() { return parsed }
718
719    return (Some(token), None)
720}
721
722fn parse_attribute_reference_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
723    if !matches!(token, Token::AttributeIdentifier) { return (Some(token), None) }
724
725    let token = guarded_unwrap!(parser.advance(), return (None, None));
726
727    if let Token::Text = token {
728        let attribute_name = parser.slice();
729        return (parser.advance(), Some(Datatype::Variant(Variant::String(format!("${}", attribute_name)))))
730    }
731
732    (Some(token), None)
733}
734
735fn resolve_static_attribute_reference<'a>(parser: &mut Parser<'a>, static_name: &str, tree_node_idx: TreeNodeType) -> Datatype {
736    let tree_node = parser.tree_nodes.get(tree_node_idx);
737
738    match tree_node {
739        AnyTreeNode::Root(node) => {
740            let resolved = node.unwrap().static_attributes.get(static_name);
741
742            match resolved {
743                Some(datatype) => datatype.clone(),
744        
745                None => Datatype::None
746            }
747        },
748
749        AnyTreeNode::Node(node) => {
750            let node = node.unwrap();
751            let resolved = node.static_attributes.get(static_name);
752
753            match resolved {
754                Some(datatype) => datatype.clone(),
755        
756                None => resolve_static_attribute_reference(parser, static_name, node.parent)
757            }
758        }
759    }
760}
761
762fn parse_static_attribute_reference_datatype<'a>(parser: &mut Parser<'a>, token: Token) -> TokenWithResult<'a, Option<Datatype>> {
763    if !matches!(token, Token::StaticAttributeIdentifier) { return (Some(token), None) }
764
765    let token = guarded_unwrap!(parser.advance(), return (None, None));
766
767    if let Token::Text = token {
768        let static_name = parser.slice();
769
770        let resolved_datatype = resolve_static_attribute_reference(parser, static_name, parser.current_tree_node_idx);
771
772        return (parser.advance(), Some(resolved_datatype))
773    }
774
775    (Some(token), None)
776}
777
778fn parse_datatype<'a>(
779    parser: &mut Parser<'a>, mut token: Token, key: Option<&str>
780) -> TokenWithResult<'a, Option<Datatype>> {
781    if let (Some(tuple_token), current_tuple_idx) = parse_tuple_name(parser, token, None, None) {
782        // Checking for a tuple can lead to a new token,
783        // even if cases where a tuple isn't found.
784        token = tuple_token;
785
786        if let Some(current_tuple_idx) = current_tuple_idx {
787            let current_tuple = parser.get_tuple(current_tuple_idx).unwrap();
788
789            return (Some(token), Some(current_tuple.coerce_to_datatype()))
790        }
791    };
792  
793    let parsed = parse_string_datatype(parser, token);
794    if parsed.1.is_some() { return parsed }
795
796    let parsed = parse_content_datatype(parser, token);
797    if parsed.1.is_some() { return parsed }
798
799    let parsed = parse_number_datatype(parser, token);
800    if parsed.1.is_some() { return parsed }
801
802    let parsed = parse_operator_datatype(parser, token);
803    if parsed.1.is_some() { return parsed }
804
805    let parsed = parse_enum_datatype(parser, token, key);
806    if parsed.1.is_some() { return parsed }
807
808    let parsed = parse_color_datatype(parser, token);
809    if parsed.1.is_some() { return parsed }
810
811    let parsed = parse_attribute_reference_datatype(parser, token);
812    if parsed.1.is_some() { return parsed }
813
814    let parsed = parse_static_attribute_reference_datatype(parser, token);
815    if parsed.1.is_some() { return parsed }
816
817    let parsed = parse_boolean_datatype(parser, token);
818    if parsed.1.is_some() { return parsed }
819
820    let parsed = parse_nil_datatype(parser, token);
821    if parsed.1.is_some() { return parsed }
822
823    (Some(token), None)
824}
825
826fn parse_datatype_group<'a>(
827    parser: &mut Parser<'a>, token: Token, key: Option<&str>,
828    mut datatype_group: Option<DatatypeGroup>, mut pending_operator: Option<Operator>
829) -> TokenWithResult<'a, Option<Datatype>> {
830    let (token, datatype) = parse_datatype(parser, token, key);
831
832    if let Some(datatype) = datatype {
833        // If the datatype is an operator then we need to postpone adding it
834        // to the datatypes group so we can atomise it with the next datatype
835        // if it is an operator.
836        if !matches!(datatype, Datatype::Operator(_)) {
837             if let Some(some_pending_operator) = pending_operator {
838                // We can add our pending operator to the datatypes table
839                // since it has no other operator to atomise with.
840                let mut datatypes_exists = DatatypeGroup::ensure_then_insert(
841                    datatype_group, 
842                    Datatype::Operator(some_pending_operator)
843                );
844                pending_operator = None;
845                datatypes_exists.push(datatype.clone());
846                datatype_group = Some(datatypes_exists);
847                
848            } else {
849                datatype_group = Some(DatatypeGroup::ensure_then_insert(datatype_group, datatype.clone()))
850            }
851        }
852
853        if let Some(some_token) = token {
854            if matches!(some_token, Token::ParensClose | Token::ScopeClose | Token::Comma | Token::SemiColon) {
855                return (token, datatype_group.and_then(|mut x| Some(x.coerce_to_datatype())));
856
857            } else {
858                if let Datatype::Operator(operator) = datatype {
859                    // Since our datatype was an operator we need to mark it as pending,
860                    // atomising with the existing pending operator if it exists.
861
862                    if let Some(some_pending_operator) = pending_operator {
863                        if some_pending_operator.can_merge_with(&operator) {
864                            pending_operator = Some(some_pending_operator.merge_with(&operator))
865                        } else {
866                            datatype_group = Some(DatatypeGroup::ensure_then_insert(datatype_group, Datatype::Operator(some_pending_operator)));
867                            pending_operator = Some(operator)
868                        }
869                    } else {
870                        pending_operator = Some(operator)
871                    }
872                }
873
874                return parse_datatype_group(parser, some_token, key, datatype_group, pending_operator);
875            }
876        } else {
877            return (token, datatype_group.and_then(|mut x| Some(x.coerce_to_datatype())));
878        }
879
880    } else {
881        return (token, datatype_group.and_then(|mut x| Some(x.coerce_to_datatype())))
882    }
883}
884
885fn parse_tuple_close<'a>(
886    parser: &mut Parser<'a>, token: Token,
887    current_tuple_idx: usize, root_tuple_idx: usize
888) -> Option<Token> {
889    if !matches!(token, Token::ParensClose) { return None }
890
891    let token = parser.advance();
892
893    let current_tuple = parser.get_tuple(current_tuple_idx).unwrap();
894    let parent_tuple_idx = current_tuple.parent;
895
896    if let Some(some_parent_tuple_idx) = parent_tuple_idx {
897        let datatype = current_tuple.coerce_to_datatype();
898        parser.get_tuple_mut(some_parent_tuple_idx).unwrap().push(datatype);
899
900        let token = guarded_unwrap!(token, return None);
901        
902        let parsed = parse_tuple_delimiter(parser, token, some_parent_tuple_idx, root_tuple_idx);
903        if parsed.is_some() { return parsed }
904
905        let parsed = parse_tuple_close(parser, token, some_parent_tuple_idx, root_tuple_idx);
906        if parsed.is_some() { return parsed }
907
908        let parsed = parse_tuple_name(
909            parser, token, parent_tuple_idx, Some(root_tuple_idx)
910        );
911        if parsed.1.is_some() { return parsed.0 }
912    };
913
914    token
915}
916
917fn parse_tuple_datatype<'a>(
918    parser: &mut Parser<'a>, token: Token,
919    current_tuple_idx: usize, root_tuple_idx: usize
920) -> Option<Token> {
921    let (token, datatype) = parse_datatype_group(
922        parser, token, None, None, None
923    );
924
925    let datatype = guarded_unwrap!(datatype, return None);
926
927    let current_tuple = parser.get_tuple_mut(current_tuple_idx).unwrap();
928    current_tuple.push(datatype);
929
930    if let Some(token) = token {
931        let parsed = parse_tuple_delimiter(parser, token, current_tuple_idx, root_tuple_idx);
932        if parsed.is_some() { return parsed }
933
934        let parsed = parse_tuple_close(parser, token, current_tuple_idx, root_tuple_idx);
935        if parsed.is_some() { return parsed }
936        
937        let parsed = parse_tuple_name(
938            parser, token, Some(current_tuple_idx), Some(root_tuple_idx)
939        );
940        if parsed.1.is_some() { return parsed.0 }
941    }
942
943    token
944}
945
946fn parse_tuple_delimiter<'a>(
947    parser: &mut Parser<'a>, token: Token,
948    current_tuple_idx: usize, root_tuple_idx: usize
949) -> Option<Token> {
950    if !matches!(token, Token::SemiColon | Token::Comma) { return None }
951
952    let token = parser.advance();
953
954    if let Some(token) = token {
955        // Handles cases of multiple delimiter tokens next to each other.
956        let parsed = parse_tuple_delimiter(parser, token, current_tuple_idx, root_tuple_idx);
957        if parsed.is_some() { return parsed }
958        
959        let parsed = parse_tuple_close(parser, token, current_tuple_idx, root_tuple_idx);
960        if parsed.is_some() { return parsed }
961
962        let parsed = parse_tuple_name(
963            parser, token, Some(current_tuple_idx), Some(root_tuple_idx)
964        );
965        if parsed.1.is_some() { return parsed.0 }
966
967        let parsed = parse_tuple_datatype(parser, token, current_tuple_idx, root_tuple_idx);
968        if parsed.is_some() { return parsed }
969    }
970
971    token
972}
973
974fn parse_tuple_open<'a>(
975    parser: &mut Parser<'a>, token: Token, tuple_name: Option<String>,
976    parent_tuple_idx: Option<usize>, root_tuple_idx: Option<usize>
977) -> TokenWithResult<'a, Option<usize>> {
978    if !matches!(token, Token::ParensOpen) { return (None, None) };
979
980    let current_tuple_idx = parser.push_tuple(
981        Tuple::new(tuple_name, parent_tuple_idx)
982    );
983
984    let token = guarded_unwrap!(parser.advance(), return (None, Some(current_tuple_idx)));
985
986    let root_tuple_idx = root_tuple_idx.unwrap_or(current_tuple_idx);
987
988    let parsed = parse_tuple_delimiter(parser, token, current_tuple_idx, root_tuple_idx);
989    if parsed.is_some() { return (parsed, Some(current_tuple_idx)) }
990
991    let parsed = parse_tuple_close(parser, token, current_tuple_idx, root_tuple_idx);
992    if parsed.is_some() { return (parsed, Some(current_tuple_idx)) }
993
994    let parsed = parse_tuple_name(
995        parser, token, Some(current_tuple_idx), Some(root_tuple_idx)
996    );
997    if parsed.0.is_some() { return (parsed.0, Some(current_tuple_idx)) }
998
999    let parsed = parse_tuple_datatype(parser, token, current_tuple_idx, root_tuple_idx);
1000    if parsed.is_some() { return (parsed, Some(current_tuple_idx)) }
1001
1002    (Some(token), Some(current_tuple_idx))
1003}
1004
1005fn parse_tuple_name<'a>(
1006    parser: &mut Parser<'a>, token: Token,
1007    parent_tuple_idx: Option<usize>, root_tuple_idx: Option<usize>
1008) -> TokenWithResult<'a, Option<usize>> {
1009    if let Token::Text = token {
1010        let tuple_name = parser.slice();
1011        let token = guarded_unwrap!(parser.advance(), return (None, None));
1012
1013        return parse_tuple_open(parser, token, Some(tuple_name.to_string()), parent_tuple_idx, root_tuple_idx)
1014
1015    } else {
1016        return parse_tuple_open(parser, token, None, parent_tuple_idx, root_tuple_idx)
1017    }
1018}
1019
1020fn parse_delimiters<'a>(parser: &mut Parser<'a>, token: Token)  -> Option<Token> {
1021    if matches!(token, Token::SemiColon | Token::Comma) {
1022        let token = parser.advance()?;
1023        return parse_delimiters(parser, token);
1024    }
1025
1026    return Some(token)
1027}
1028
1029fn parse_property<'a>(parser: &mut Parser<'a>, mut token: Token) -> Option<Token> {
1030    if let Token::Text = token {
1031        let property_name = parser.slice();
1032        let selector_token = token;
1033
1034        token = parser.advance()?;
1035
1036        if !matches!(token, Token::Equals) {
1037            return parse_scope_selector(parser, token, Selector::new(property_name, selector_token))
1038        };
1039
1040        // We only want to parse the property if the current node is not the root.
1041        if let TreeNodeType::Node(node_idx) = parser.current_tree_node_idx {
1042            token = parser.advance()?;
1043
1044            let (token, datatype) = parse_datatype_group(
1045                parser, token, Some(property_name), None, None
1046            );
1047            let variant = datatype.and_then(|d| d.coerce_to_variant(Some(property_name)));
1048    
1049            if let Some(variant) = variant {
1050                let current_tree_node = parser.tree_nodes[node_idx].as_mut().unwrap();
1051                current_tree_node.properties.insert(property_name.to_string(), variant);
1052            }
1053    
1054            let token = guarded_unwrap!(token, return None);
1055            return parse_delimiters(parser, token);
1056
1057        } else {
1058            return Some(token)
1059        }
1060    }
1061
1062    Some(token)
1063}
1064
1065fn parse_static_attribute<'a>(parser: &mut Parser<'a>, mut token: Token) -> Option<Token> {
1066    if !matches!(token, Token::StaticAttributeIdentifier) { return Some(token) };
1067
1068    token = parser.advance()?;
1069
1070    if let Token::Text = token {
1071        let static_name = parser.slice();
1072        let next_token = parser.advance()?;
1073        
1074        if !matches!(next_token, Token::Equals) { return Some(next_token) }
1075
1076        token = parser.advance()?;
1077
1078        let (token, datatype) = parse_datatype_group(
1079            parser, token, Some(static_name), None, None
1080        );
1081        let datatype = datatype.and_then(|d| d.coerce_to_static(Some(static_name)));
1082
1083        if let Some(datatype) = datatype {
1084            let current_tree_node = parser.tree_nodes.get_node_mut(parser.current_tree_node_idx);
1085            match current_tree_node {
1086                AnyTreeNodeMut::Root(node) => {
1087                    node.unwrap().static_attributes.insert(static_name.to_string(), datatype)
1088                },
1089                AnyTreeNodeMut::Node(node) => {
1090                    node.unwrap().static_attributes.insert(static_name.to_string(), datatype)
1091                }
1092            };
1093        }
1094
1095        let token = guarded_unwrap!(token, return None);
1096        return parse_delimiters(parser, token)
1097    }
1098
1099    Some(token)
1100}
1101
1102fn parse_attribute<'a>(parser: &mut Parser<'a>, mut token: Token) -> Option<Token> {
1103    if !matches!(token, Token::AttributeIdentifier) { return Some(token) };
1104
1105    token = parser.advance()?;
1106
1107    if let Token::Text = token {
1108        let attribute_name = parser.slice();
1109        let next_token = parser.advance()?;
1110        
1111        if !matches!(next_token, Token::Equals) { return Some(next_token) }
1112
1113        token = parser.advance()?;
1114
1115        let (token, datatype) = parse_datatype_group(
1116            parser, token, Some(attribute_name), None, None
1117        );
1118        let variant = datatype.and_then(|d| d.coerce_to_variant(Some(attribute_name)));
1119
1120        if let Some(variant) = variant {
1121            let current_tree_node = parser.tree_nodes.get_node_mut(parser.current_tree_node_idx);
1122            match current_tree_node {
1123                AnyTreeNodeMut::Root(node) => {
1124                    node.unwrap().attributes.insert(attribute_name.to_string(), variant)
1125                },
1126                AnyTreeNodeMut::Node(node) => {
1127                    node.unwrap().attributes.insert(attribute_name.to_string(), variant)
1128                }
1129            };
1130        }
1131
1132        let token = guarded_unwrap!(token, return None);
1133        return parse_delimiters(parser, token)
1134    }
1135
1136    Some(token)
1137}
1138
1139fn parse_priority_declaration<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1140    if !matches!(token, Token::PriorityDeclaration) { return Some(token) }
1141
1142    let token = parser.advance()?;
1143
1144    // We only want to parse the priority if the current node is not the root.
1145    if let TreeNodeType::Node(node_idx) = parser.current_tree_node_idx {
1146        let (token, datatype) = parse_datatype_group(parser, token, None, None, None);
1147
1148        if let Some(Datatype::Variant(Variant::Float32(float32))) = datatype {
1149            let current_tree_node = parser.tree_nodes[node_idx].as_mut().unwrap();
1150            current_tree_node.priority = Some(float32 as i32);
1151        }
1152    
1153        let token = guarded_unwrap!(token, return None);
1154        parse_delimiters(parser, token)
1155
1156    } else {
1157        Some(token)
1158    }
1159}
1160
1161fn parse_name_declaration<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1162    if !matches!(token, Token::NameDeclaration) { return Some(token) }
1163
1164    let token = parser.advance()?;
1165
1166    // We only want to parse the name if the current node is not the root.
1167    if let TreeNodeType::Node(node_idx) = parser.current_tree_node_idx {
1168        let (token, datatype) = parse_datatype_group(
1169            parser, token, None, None, None
1170        );
1171    
1172        if let Some(Datatype::Variant(Variant::String(name))) = datatype {
1173            let current_tree_node = parser.tree_nodes[node_idx].as_mut().unwrap();
1174            current_tree_node.name = Some(name);
1175        }
1176    
1177        let token = guarded_unwrap!(token, return None);
1178        parse_delimiters(parser, token)
1179
1180    } else {
1181        Some(token)
1182    }
1183}
1184
1185fn parse_ignore_derive_declaration<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1186    if !matches!(token, Token::DeriveDeclaration) { return Some(token) }
1187
1188    let token = parser.core_advance()?;
1189    if !matches!(token, Token::Text) { return Some(token) }
1190
1191    let token = parser.core_advance()?;
1192
1193    parse_ignore_tuple_open(parser, token)
1194}
1195
1196// Util and Macro declarations are ignored in the main parser.
1197fn parse_ignore_scope_open<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1198    if !matches!(token, Token::ScopeOpen) { return Some(token) }
1199
1200    let mut nestedness = 0usize;
1201
1202    loop {
1203        match parser.core_advance()? {
1204            Token::ScopeOpen => nestedness += 1,
1205            Token::ScopeClose => match nestedness {
1206                // End of parsing.
1207                0 => return parser.advance(),
1208                _ => nestedness -= 1
1209            },
1210            _ => ()
1211        }
1212    }
1213}
1214
1215// Util declarations are ignored in the main parser.
1216fn parse_util_declaration<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1217    if !matches!(token, Token::UtilDeclaration) { return Some(token) }
1218
1219    let token = parser.core_advance()?;
1220    if !matches!(token, Token::Text) { return Some(token) }
1221
1222    let token = parser.core_advance()?;
1223    parse_ignore_scope_open(parser, token)
1224}
1225
1226// Macro declarations are ignored in the main parser.
1227fn parse_ignore_tuple_open<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1228    if !matches!(token, Token::ParensOpen) { return Some(token) }
1229
1230    let mut nestedness = 0usize;
1231
1232    loop {
1233        match parser.core_advance()? {
1234            Token::ParensOpen => nestedness += 1,
1235            Token::ParensClose => match nestedness {
1236                // End of parsing.
1237                0 => return parser.advance(),
1238                _ => nestedness -= 1
1239            },
1240            _ => ()
1241        }
1242    }
1243}
1244
1245// Macro declarations are ignored in the main parser.
1246fn parse_macro_declaration<'a>(parser: &mut Parser<'a>, token: Token) -> Option<Token> {
1247    if !matches!(token, Token::MacroDeclaration) { return Some(token) }
1248
1249    let token = parser.core_advance()?;
1250    if !matches!(token, Token::Text) { return Some(token) }
1251
1252    let token = parser.core_advance()?;
1253
1254    let token = parse_ignore_tuple_open(parser, token)?;
1255
1256    parse_ignore_scope_open(parser, token)
1257}
1258
1259fn main_loop<'a>(parser: &mut Parser<'a>) -> Option<()> {
1260    let mut token = parser.advance()?;
1261
1262    loop {
1263        parser.did_advance = false;
1264
1265        token = parse_attribute(parser, token)?;
1266        token = parse_static_attribute(parser, token)?;
1267        token = parse_property(parser, token)?;
1268        token = parse_scope_selector_start(parser, token)?;
1269        token = parse_scope_open(parser, token, None)?;
1270        token = parse_scope_close(parser, token)?;
1271        token = parse_priority_declaration(parser, token)?;
1272        token = parse_name_declaration(parser, token)?;
1273        token = parse_ignore_derive_declaration(parser, token)?;
1274        token = parse_util_declaration(parser, token)?;
1275        token = parse_macro_declaration(parser, token)?;
1276
1277        // Ensures the parser is advanced at least one time per iteration.
1278        // This prevents infinite loops.
1279        if !parser.did_advance {
1280            token = guarded_unwrap!(parser.advance(), break)
1281        }
1282    }
1283
1284    None
1285}
1286
1287pub fn parse_rsml<'a>(lexer: &'a mut logos::Lexer<'a, Token>, macros: &'a MacroGroup) -> TreeNodeGroup {
1288    let mut parser = Parser::<'a>::new(lexer, macros);
1289
1290    main_loop(&mut parser);
1291
1292    return parser.tree_nodes;
1293}
rbx_rsml/parser/mod.rs

rbx_rsml/parser/
mod.rs
