mnk_vmf/parser/
util.rs

1use std::vec::IntoIter;
2
3use chumsky::input::Stream;
4use logos::Logos as _;
5
6use super::lexer;
7
8/// Macro to define individual property parsers and combine them with .or().
9/// When this Macro is used, it is necrssary to have chumsky's .or() and .map() in the scope.
10///
11/// Usage:
12/// ```ignore
13/// impl_block_properties_parser! {
14///     // Output variable name for the combined parser: Its type will be Box<dyn Parser<..., Output = $PropEnumType, ...>>
15///     any_property_variable_name: YourPropertyEnumType = {
16///         // var_name = parser_call_expr => enum_variant_constructor_or_mapper_fn
17///         p_some_bool = some_parser_that_outputs_bool("some_key") => YourPropertyEnumType::SomeBool,
18///         p_some_num  = some_parser_that_outputs_u32("num_key")  => YourPropertyEnumType::SomeNum,
19///         // ...
20///     }
21/// }
22/// ```
23#[macro_export]
24macro_rules! impl_block_properties_parser {
25    (@build_or_chain $first_parser_var:ident) => {
26        $first_parser_var
27    };
28    (@build_or_chain $first_parser_var:ident, $($rest_parser_vars:ident),+) => {
29        $first_parser_var.or(impl_block_properties_parser!(@build_or_chain $($rest_parser_vars),+))
30    };
31
32    (
33        $any_property_let_name:ident: $PropEnumType:ty = {
34            $(
35                $var_name:ident = $parser_call_expr:expr => $value_mapper_fn:expr
36            ),+ $(,)? // Allow trailing comma
37        }
38    ) => {
39        $(
40            let $var_name = $parser_call_expr.map($value_mapper_fn);
41        )+
42        let $any_property_let_name =
43            impl_block_properties_parser!(@build_or_chain $($var_name),+).boxed();
44    };
45}
46
47/// Helper function (nostly for tests and benchmarks) to get Token stream out of input
48pub fn lex(input: &str) -> Stream<IntoIter<lexer::Token<'_>>> {
49    Stream::from_iter(
50        lexer::Token::lexer(input)
51            .map(|tok| tok.expect("expected a valid token."))
52            .collect::<Vec<lexer::Token<'_>>>(),
53    )
54}
55
56/// Produces a vector of tokens (for reuse or benchmarking).
57pub fn tokenize(input: &str) -> Vec<lexer::Token<'_>> {
58    lexer::Token::lexer(input)
59        .enumerate()
60        .map(|(idx, tok)| {
61            tok.unwrap_or_else(|_| {
62                let lexer = lexer::Token::lexer(input);
63                let span = lexer.span();
64                let context = &input
65                    [span.start.saturating_sub(20)..span.end.saturating_add(20).min(input.len())];
66                panic!(
67                    "Failed to tokenize at position {} (token #{})\nContext: {:?}",
68                    span.start, idx, context
69                );
70            })
71        })
72        .collect()
73}
74
75/// Wraps tokens into a Stream that Chumsky can parse.
76pub fn stream(tokens: Vec<lexer::Token<'_>>) -> Stream<IntoIter<lexer::Token<'_>>> {
77    Stream::from_iter(tokens)
78}
79
80#[cfg(test)]
81mod tests {
82    use super::*;
83
84    #[test]
85    fn test_tokenize_simple() {
86        let input = r#"world
87    {
88        "id" "1"
89    }"#;
90        let tokens = tokenize(input);
91        println!("Tokens: {:#?}", tokens);
92    }
93
94    #[test]
95    fn test_tokenize_with_special_chars() {
96        let input = r#""detail/detailsprites""#;
97        let tokens = tokenize(input);
98        println!("Tokens: {:#?}", tokens);
99    }
100
101    #[test]
102    fn test_tokenize_with_brackets() {
103        let input = r#""[1 0 0 0]""#;
104        let tokens = tokenize(input);
105        println!("Tokens: {:#?}", tokens);
106    }
107
108    #[test]
109    fn test_tokenize_negative_number() {
110        let input = r#""-1""#;
111        let tokens = tokenize(input);
112        println!("Tokens: {:#?}", tokens);
113    }
114}
mnk_vmf/parser/util.rs

mnk_vmf/parser/
util.rs
