use std::vec::IntoIter;
use chumsky::input::Stream;
use logos::Logos as _;
use super::lexer;
#[macro_export]
macro_rules! impl_block_properties_parser {
(@build_or_chain $first_parser_var:ident) => {
$first_parser_var
};
(@build_or_chain $first_parser_var:ident, $($rest_parser_vars:ident),+) => {
$first_parser_var.or(impl_block_properties_parser!(@build_or_chain $($rest_parser_vars),+))
};
(
$any_property_let_name:ident: $PropEnumType:ty = {
$(
$var_name:ident = $parser_call_expr:expr => $value_mapper_fn:expr
),+ $(,)? // Allow trailing comma
}
) => {
$(
let $var_name = $parser_call_expr.map($value_mapper_fn);
)+
let $any_property_let_name =
impl_block_properties_parser!(@build_or_chain $($var_name),+).boxed();
};
}
pub fn lex(input: &str) -> Stream<IntoIter<lexer::Token<'_>>> {
Stream::from_iter(
lexer::Token::lexer(input)
.map(|tok| tok.expect("expected a valid token."))
.collect::<Vec<lexer::Token<'_>>>(),
)
}
pub fn tokenize(input: &str) -> Vec<lexer::Token<'_>> {
lexer::Token::lexer(input)
.enumerate()
.map(|(idx, tok)| {
tok.unwrap_or_else(|_| {
let lexer = lexer::Token::lexer(input);
let span = lexer.span();
let context = &input
[span.start.saturating_sub(20)..span.end.saturating_add(20).min(input.len())];
panic!(
"Failed to tokenize at position {} (token #{})\nContext: {:?}",
span.start, idx, context
);
})
})
.collect()
}
pub fn stream(tokens: Vec<lexer::Token<'_>>) -> Stream<IntoIter<lexer::Token<'_>>> {
Stream::from_iter(tokens)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_tokenize_simple() {
let input = r#"world
{
"id" "1"
}"#;
let tokens = tokenize(input);
println!("Tokens: {:#?}", tokens);
}
#[test]
fn test_tokenize_with_special_chars() {
let input = r#""detail/detailsprites""#;
let tokens = tokenize(input);
println!("Tokens: {:#?}", tokens);
}
#[test]
fn test_tokenize_with_brackets() {
let input = r#""[1 0 0 0]""#;
let tokens = tokenize(input);
println!("Tokens: {:#?}", tokens);
}
#[test]
fn test_tokenize_negative_number() {
let input = r#""-1""#;
let tokens = tokenize(input);
println!("Tokens: {:#?}", tokens);
}
}