use super::*;
use chumsky::{
extra::{Err, ParserExtra},
input::{BorrowInput, ValueInput},
prelude::*,
};
#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum TokenTree {
Lone(Token),
Parens(Box<Vec<(TokenTree, Span32)>>),
Brackets(Box<Vec<(TokenTree, Span32)>>),
Braces(Box<Vec<(TokenTree, Span32)>>),
#[default]
TreeError,
}
#[test]
fn test_token_tree_size() {
assert_eq!(size_of::<TokenTree>(), size_of::<[usize; 2]>());
assert_eq!(size_of::<(TokenTree, Span32)>(), size_of::<[usize; 3]>());
}
pub fn trees_of<'src>(
tokens: &'src [(Token, Span32)],
) -> (Vec<(TokenTree, Span32)>, Vec<Rich<'src, Token, Span32>>) {
let eoi: Span32 = match tokens.last() {
Some(s) => s.1,
None => return (Vec::new(), Vec::new()),
};
let recovery = via_parser(
any()
.repeated()
.at_least(1)
.to(TokenTree::TreeError)
.map_with(|tree, ex| (tree, ex.span())),
);
let tree_parser =
token_tree_p().recover_with(recovery).repeated().collect::<Vec<_>>();
let (opt_out, errors) = tree_parser
.parse(Input::map(tokens, eoi, |(tk, span)| (tk, span)))
.into_output_errors();
(opt_out.unwrap_or_default(), errors)
}
fn assert_output<'src, O, P, I, E>(_: &P)
where
P: Parser<'src, I, O, E>,
I: Input<'src>,
E: ParserExtra<'src, I>,
{
}
fn token_tree_p<'src, I>()
-> impl Parser<'src, I, (TokenTree, Span32), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
recursive(|tokens| {
let comment = {
let line_comment = line_comment_p();
let block_comment = tokens
.clone()
.repeated()
.delimited_by(open_comment_p(), close_comment_p())
.ignored();
choice((line_comment, block_comment))
};
let braces = tokens
.clone()
.repeated()
.collect::<Vec<_>>()
.then_ignore(comment.clone().repeated())
.delimited_by(open_brace_p(), close_brace_p())
.map_with(|out, ex| (TokenTree::Braces(Box::new(out)), ex.span()));
let brackets = tokens
.clone()
.repeated()
.collect::<Vec<_>>()
.then_ignore(comment.clone().repeated())
.delimited_by(open_bracket_p(), close_bracket_p())
.map_with(|out, ex| (TokenTree::Brackets(Box::new(out)), ex.span()));
let parens = tokens
.clone()
.repeated()
.collect::<Vec<_>>()
.then_ignore(comment.clone().repeated())
.delimited_by(open_paren_p(), close_paren_p())
.map_with(|out, ex| (TokenTree::Parens(Box::new(out)), ex.span()));
let lone =
non_tree_token_p().map_with(|out, ex| (TokenTree::Lone(out), ex.span()));
assert_output::<(TokenTree, Span32), _, _, _>(&lone);
let x = choice((brackets, braces, parens, lone))
.padded_by(comment.clone().repeated());
x
})
}
fn open_brace_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::OpBrace => (),
}
.labelled("`{`")
.as_context()
}
fn close_brace_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::ClBrace => (),
}
.labelled("`}`")
.as_context()
}
fn open_bracket_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::OpBracket => (),
}
.labelled("`[`")
.as_context()
}
fn close_bracket_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::ClBracket => (),
}
.labelled("`]`")
.as_context()
}
fn open_paren_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::OpParen => (),
}
.labelled("`(`")
.as_context()
}
fn close_paren_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::ClParen => (),
}
.labelled("`)`")
.as_context()
}
fn open_comment_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::OpBlockComment => (),
}
.labelled("`/*`")
.as_context()
}
fn close_comment_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::ClBlockComment => (),
}
.labelled("`*/`")
.as_context()
}
fn non_tree_token_p<'src, I>()
-> impl Parser<'src, I, Token, Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
none_of([
Token::OpBracket,
Token::ClBracket,
Token::OpBrace,
Token::ClBrace,
Token::OpParen,
Token::ClParen,
Token::OpBlockComment,
Token::ClBlockComment,
])
.labelled("non_tree_token")
.as_context()
}
fn line_comment_p<'src, I>()
-> impl Parser<'src, I, (), Err<Rich<'src, Token, Span32>>> + Clone
where
I: BorrowInput<'src, Token = Token, Span = Span32> + ValueInput<'src>,
{
select! {
Token::LineComment => (),
Token::DocComment => (),
Token::InteriorComment => (),
}
.labelled("line_comment")
.as_context()
}