pub mod owned;
#[macro_use]
mod parser_util;
mod parsers;
pub mod punctuated;
pub mod span;
use crate::tokenizer::{Symbol, Token, TokenKind, TokenReference, TokenType};
use full_moon_derive::{Node, Owned, Visit};
use generational_arena::Arena;
use itertools::Itertools;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use std::{borrow::Cow, fmt, iter::FromIterator, sync::Arc};
use parser_util::{
InternalAstError, OneOrMore, Parser, ParserState, ZeroOrMore, ZeroOrMoreDelimited,
};
use punctuated::{Pair, Punctuated};
use span::ContainedSpan;
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct Block<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
stmts: Vec<(Stmt<'a>, Option<TokenReference<'a>>)>,
#[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
last_stmt: Option<(LastStmt<'a>, Option<TokenReference<'a>>)>,
}
impl<'a> Block<'a> {
pub fn iter_stmts(&self) -> impl Iterator<Item = &Stmt<'a>> {
self.stmts.iter().map(|(stmt, _)| stmt)
}
pub fn last_stmts(&self) -> Option<&LastStmt<'a>> {
Some(&self.last_stmt.as_ref()?.0)
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum LastStmt<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Break(TokenReference<'a>),
Return(Return<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct Return<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
token: TokenReference<'a>,
returns: Punctuated<'a, Expression<'a>>,
}
impl<'a> Return<'a> {
pub fn token(&self) -> &TokenReference<'a> {
&self.token
}
pub fn returns(&self) -> &Punctuated<'a, Expression<'a>> {
&self.returns
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Field<'a> {
ExpressionKey {
#[cfg_attr(feature = "serde", serde(borrow))]
brackets: ContainedSpan<'a>,
key: Expression<'a>,
equal: TokenReference<'a>,
value: Expression<'a>,
},
NameKey {
#[cfg_attr(feature = "serde", serde(borrow))]
key: TokenReference<'a>,
equal: TokenReference<'a>,
value: Expression<'a>,
},
#[cfg_attr(feature = "serde", serde(borrow))]
NoKey(Expression<'a>),
}
pub type TableConstructorField<'a> = (Field<'a>, Option<TokenReference<'a>>);
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct TableConstructor<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
braces: ContainedSpan<'a>,
fields: Vec<TableConstructorField<'a>>,
}
impl<'a> TableConstructor<'a> {
pub fn braces(&self) -> &ContainedSpan<'a> {
&self.braces
}
pub fn iter_fields(&self) -> impl Iterator<Item = &TableConstructorField<'a>> {
self.fields.iter()
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[visit(visit_as = "bin_op")]
pub struct BinOpRhs<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
bin_op: BinOp<'a>,
rhs: Box<Expression<'a>>,
}
impl<'a> BinOpRhs<'a> {
pub fn bin_op(&self) -> &BinOp<'a> {
&self.bin_op
}
pub fn rhs(&self) -> &Expression<'a> {
self.rhs.as_ref()
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[cfg_attr(feature = "serde", serde(untagged))]
pub enum Expression<'a> {
Parentheses {
#[cfg_attr(feature = "serde", serde(borrow))]
contained: ContainedSpan<'a>,
expression: Box<Expression<'a>>,
},
UnaryOperator {
#[cfg_attr(feature = "serde", serde(borrow))]
unop: UnOp<'a>,
expression: Box<Expression<'a>>,
},
Value {
#[cfg_attr(feature = "serde", serde(borrow))]
value: Box<Value<'a>>,
binop: Option<BinOpRhs<'a>>,
},
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Value<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Function((TokenReference<'a>, FunctionBody<'a>)),
FunctionCall(FunctionCall<'a>),
TableConstructor(TableConstructor<'a>),
Number(TokenReference<'a>),
ParseExpression(Expression<'a>),
String(TokenReference<'a>),
Symbol(TokenReference<'a>),
Var(Var<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Stmt<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Assignment(Assignment<'a>),
Do(Do<'a>),
FunctionCall(FunctionCall<'a>),
FunctionDeclaration(FunctionDeclaration<'a>),
GenericFor(GenericFor<'a>),
If(If<'a>),
LocalAssignment(LocalAssignment<'a>),
LocalFunction(LocalFunction<'a>),
NumericFor(NumericFor<'a>),
Repeat(Repeat<'a>),
While(While<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Prefix<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Expression(Expression<'a>),
Name(TokenReference<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Index<'a> {
Brackets {
#[cfg_attr(feature = "serde", serde(borrow))]
brackets: ContainedSpan<'a>,
expression: Expression<'a>,
},
Dot {
#[cfg_attr(feature = "serde", serde(borrow))]
dot: TokenReference<'a>,
name: TokenReference<'a>,
},
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum FunctionArgs<'a> {
Parentheses {
#[cfg_attr(feature = "serde", serde(borrow))]
arguments: Punctuated<'a, Expression<'a>>,
parentheses: ContainedSpan<'a>,
},
#[cfg_attr(feature = "serde", serde(borrow))]
String(TokenReference<'a>),
TableConstructor(TableConstructor<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct NumericFor<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
for_token: TokenReference<'a>,
index_variable: TokenReference<'a>,
equal_token: TokenReference<'a>,
start: Expression<'a>,
start_end_comma: TokenReference<'a>,
end: Expression<'a>,
end_step_comma: Option<TokenReference<'a>>,
step: Option<Expression<'a>>,
do_token: TokenReference<'a>,
block: Block<'a>,
end_token: TokenReference<'a>,
}
impl<'a> NumericFor<'a> {
pub fn for_token(&self) -> &TokenReference<'a> {
&self.for_token
}
pub fn index_variable(&self) -> &TokenReference<'a> {
&self.index_variable
}
pub fn equal_token(&self) -> &TokenReference<'a> {
&self.equal_token
}
pub fn start(&self) -> &Expression<'a> {
&self.start
}
pub fn start_end_comma(&self) -> &TokenReference<'a> {
&self.start_end_comma
}
pub fn end(&self) -> &Expression<'a> {
&self.end
}
pub fn end_step_comma(&self) -> Option<&TokenReference<'a>> {
self.end_step_comma.as_ref()
}
pub fn step(&self) -> Option<&Expression<'a>> {
self.step.as_ref()
}
pub fn do_token(&self) -> &TokenReference<'a> {
&self.do_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn end_token(&self) -> &TokenReference<'a> {
&self.end_token
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct GenericFor<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
for_token: TokenReference<'a>,
names: Punctuated<'a, TokenReference<'a>>,
in_token: TokenReference<'a>,
expr_list: Punctuated<'a, Expression<'a>>,
do_token: TokenReference<'a>,
block: Block<'a>,
end_token: TokenReference<'a>,
}
impl<'a> GenericFor<'a> {
pub fn for_token(&self) -> &TokenReference<'a> {
&self.for_token
}
pub fn names(&self) -> &Punctuated<'a, TokenReference<'a>> {
&self.names
}
pub fn in_token(&self) -> &TokenReference<'a> {
&self.in_token
}
pub fn expr_list(&self) -> &Punctuated<'a, Expression<'a>> {
&self.expr_list
}
pub fn do_token(&self) -> &TokenReference<'a> {
&self.do_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn end_token(&self) -> &TokenReference<'a> {
&self.end_token
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct If<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
if_token: TokenReference<'a>,
condition: Expression<'a>,
then_token: TokenReference<'a>,
block: Block<'a>,
else_if: Option<Vec<ElseIf<'a>>>,
else_token: Option<TokenReference<'a>>,
#[cfg_attr(feature = "serde", serde(rename = "else"))]
r#else: Option<Block<'a>>,
end_token: TokenReference<'a>,
}
impl<'a> If<'a> {
pub fn if_token(&self) -> &TokenReference<'a> {
&self.if_token
}
pub fn condition(&self) -> &Expression<'a> {
&self.condition
}
pub fn then_token(&self) -> &TokenReference<'a> {
&self.then_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn else_token(&self) -> Option<&TokenReference<'a>> {
self.else_token.as_ref()
}
pub fn else_if(&self) -> Option<&Vec<ElseIf<'a>>> {
self.else_if.as_ref()
}
pub fn else_block(&self) -> Option<&Block<'a>> {
self.r#else.as_ref()
}
pub fn end_token(&self) -> &TokenReference<'a> {
&self.end_token
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct ElseIf<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
else_if_token: TokenReference<'a>,
condition: Expression<'a>,
then_token: TokenReference<'a>,
block: Block<'a>,
}
impl<'a> ElseIf<'a> {
pub fn else_if_token(&self) -> &TokenReference<'a> {
&self.else_if_token
}
pub fn condition(&self) -> &Expression<'a> {
&self.condition
}
pub fn then_token(&self) -> &TokenReference<'a> {
&self.then_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct While<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
while_token: TokenReference<'a>,
condition: Expression<'a>,
do_token: TokenReference<'a>,
block: Block<'a>,
end_token: TokenReference<'a>,
}
impl<'a> While<'a> {
pub fn while_token(&self) -> &TokenReference<'a> {
&self.while_token
}
pub fn condition(&self) -> &Expression<'a> {
&self.condition
}
pub fn do_token(&self) -> &TokenReference<'a> {
&self.do_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn end_token(&self) -> &TokenReference<'a> {
&self.end_token
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct Repeat<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
repeat_token: TokenReference<'a>,
block: Block<'a>,
until_token: TokenReference<'a>,
until: Expression<'a>,
}
impl<'a> Repeat<'a> {
pub fn repeat_token(&self) -> &TokenReference<'a> {
&self.repeat_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn until_token(&self) -> &TokenReference<'a> {
&self.until_token
}
pub fn until(&self) -> &Expression<'a> {
&self.until
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct MethodCall<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
colon_token: TokenReference<'a>,
name: TokenReference<'a>,
args: FunctionArgs<'a>,
}
impl<'a> MethodCall<'a> {
pub fn colon_token(&self) -> &TokenReference<'a> {
&self.colon_token
}
pub fn args(&self) -> &FunctionArgs<'a> {
&self.args
}
pub fn name(&self) -> &TokenReference<'a> {
&self.name
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Call<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
AnonymousCall(FunctionArgs<'a>),
MethodCall(MethodCall<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct FunctionBody<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
parameters_parantheses: ContainedSpan<'a>,
parameters: Punctuated<'a, Parameter<'a>>,
block: Block<'a>,
end_token: TokenReference<'a>,
}
impl<'a> FunctionBody<'a> {
pub fn parameters_parantheses(&self) -> &ContainedSpan<'a> {
&self.parameters_parantheses
}
pub fn iter_parameters(&self) -> impl Iterator<Item = &Parameter<'a>> {
self.parameters.iter()
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn end_token(&self) -> &TokenReference<'a> {
&self.end_token
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Parameter<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Ellipse(TokenReference<'a>),
Name(TokenReference<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Suffix<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Call(Call<'a>),
Index(Index<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct VarExpression<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
prefix: Prefix<'a>,
suffixes: Vec<Suffix<'a>>,
}
impl<'a> VarExpression<'a> {
pub fn prefix(&self) -> &Prefix<'a> {
&self.prefix
}
pub fn iter_suffixes(&self) -> impl Iterator<Item = &Suffix<'a>> {
self.suffixes.iter()
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum Var<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
Expression(VarExpression<'a>),
Name(TokenReference<'a>),
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct Assignment<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
var_list: Punctuated<'a, Var<'a>>,
equal_token: TokenReference<'a>,
expr_list: Punctuated<'a, Expression<'a>>,
}
impl<'a> Assignment<'a> {
pub fn expr_list(&self) -> &Punctuated<'a, Expression<'a>> {
&self.expr_list
}
pub fn equal_token(&self) -> &TokenReference<'a> {
&self.equal_token
}
pub fn var_list(&self) -> &Punctuated<'a, Var<'a>> {
&self.var_list
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct LocalFunction<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
local_token: TokenReference<'a>,
function_token: TokenReference<'a>,
name: TokenReference<'a>,
func_body: FunctionBody<'a>,
}
impl<'a> LocalFunction<'a> {
pub fn local_token(&self) -> &TokenReference<'a> {
&self.local_token
}
pub fn function_token(&self) -> &TokenReference<'a> {
&self.function_token
}
pub fn func_body(&self) -> &FunctionBody<'a> {
&self.func_body
}
pub fn name(&self) -> &TokenReference<'a> {
&self.name
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct LocalAssignment<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
local_token: TokenReference<'a>,
name_list: Punctuated<'a, TokenReference<'a>>,
equal_token: Option<TokenReference<'a>>,
expr_list: Punctuated<'a, Expression<'a>>,
}
impl<'a> LocalAssignment<'a> {
pub fn local_token(&self) -> &TokenReference<'a> {
&self.local_token
}
pub fn equal_token(&self) -> Option<&TokenReference<'a>> {
self.equal_token.as_ref()
}
pub fn expr_list(&self) -> &Punctuated<'a, Expression<'a>> {
&self.expr_list
}
pub fn name_list(&self) -> &Punctuated<'a, TokenReference<'a>> {
&self.name_list
}
pub fn name_list_mut(&mut self) -> &mut Punctuated<'a, TokenReference<'a>> {
&mut self.name_list
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct Do<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
do_token: TokenReference<'a>,
block: Block<'a>,
end_token: TokenReference<'a>,
}
impl<'a> Do<'a> {
pub fn do_token(&self) -> &TokenReference<'a> {
&self.do_token
}
pub fn block(&self) -> &Block<'a> {
&self.block
}
pub fn end_token(&self) -> &TokenReference<'a> {
&self.end_token
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct FunctionCall<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
prefix: Prefix<'a>,
suffixes: Vec<Suffix<'a>>,
}
impl<'a> FunctionCall<'a> {
pub fn prefix(&self) -> &Prefix<'a> {
&self.prefix
}
pub fn iter_suffixes(&self) -> impl Iterator<Item = &Suffix<'a>> {
self.suffixes.iter()
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct FunctionName<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
names: Punctuated<'a, TokenReference<'a>>,
colon_name: Option<(TokenReference<'a>, TokenReference<'a>)>,
}
impl<'a> FunctionName<'a> {
pub fn method_name(&self) -> Option<&TokenReference<'a>> {
Some(&self.colon_name.as_ref()?.1)
}
pub fn names(&self) -> &Punctuated<'a, TokenReference<'a>> {
&self.names
}
}
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct FunctionDeclaration<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
function_token: TokenReference<'a>,
name: FunctionName<'a>,
body: FunctionBody<'a>,
}
impl<'a> FunctionDeclaration<'a> {
pub fn function_token(&self) -> &TokenReference<'a> {
&self.function_token
}
pub fn body(&self) -> &FunctionBody<'a> {
&self.body
}
pub fn name(&self) -> &FunctionName<'a> {
&self.name
}
}
macro_rules! make_op {
($enum:ident, $(#[$outer:meta])* { $($operator:ident,)+ }) => {
#[derive(Clone, Debug, PartialEq, Owned, Node, Visit)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[visit(skip_visit_self)]
$(#[$outer])*
pub enum $enum<'a> {
#[cfg_attr(feature = "serde", serde(borrow))]
$(
#[allow(missing_docs)]
$operator(TokenReference<'a>),
)+
}
};
}
make_op!(BinOp,
#[doc = "Operators that require two operands, such as X + Y or X - Y"]
{
And,
Caret,
GreaterThan,
GreaterThanEqual,
LessThan,
LessThanEqual,
Minus,
Or,
Percent,
Plus,
Slash,
Star,
TildeEqual,
TwoDots,
TwoEqual,
}
);
make_op!(UnOp,
#[doc = "Operators that require just one operand, such as #X"]
{
Minus,
Not,
Hash,
}
);
#[derive(Clone, Debug, PartialEq)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub enum AstError<'a> {
Empty,
NoEof,
UnexpectedToken {
#[cfg_attr(feature = "serde", serde(borrow))]
token: Token<'a>,
additional: Option<Cow<'a, str>>,
},
}
impl<'a> fmt::Display for AstError<'a> {
fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
match self {
AstError::Empty => write!(formatter, "tokens passed was empty, which shouldn't happen normally"),
AstError::NoEof => write!(formatter, "tokens passed had no eof token, which shouldn't happen normally"),
AstError::UnexpectedToken { token, additional } => write!(
formatter,
"unexpected token `{}`. (starting from line {}, character {} and ending on line {}, character {}){}",
token,
token.start_position().line(),
token.start_position().character(),
token.end_position().line(),
token.end_position().character(),
match additional {
Some(additional) => format!("\nadditional information: {}", additional),
None => String::new(),
}
)
}
}
}
impl<'a> std::error::Error for AstError<'a> {}
#[derive(Clone, Debug)]
pub struct Ast<'a> {
nodes: Block<'a>,
pub(crate) tokens: Arc<Arena<Token<'a>>>,
}
impl<'a> Ast<'a> {
pub fn from_tokens(tokens: Vec<Token<'a>>) -> Result<Ast<'a>, AstError<'a>> {
if *tokens.last().ok_or(AstError::Empty)?.token_type() != TokenType::Eof {
Err(AstError::NoEof)
} else {
let tokens = Arc::new(Arena::from_iter(tokens));
let mut state = ParserState::new(Arc::clone(&tokens));
if tokens
.iter()
.filter(|token| !token.1.token_type().ignore())
.count()
== 1
{
return Ok(Ast {
nodes: Block {
stmts: Vec::new(),
last_stmt: None,
},
tokens,
});
}
if state.peek().token_type().ignore() {
state = state.advance().unwrap();
}
match parsers::ParseBlock.parse(state.clone()) {
Ok((state, block)) => {
if state.index == tokens.len() - 1 {
Ok(Ast {
tokens,
nodes: block,
})
} else {
Err(AstError::UnexpectedToken {
token: (*state.peek()).to_owned(),
additional: Some(Cow::Borrowed("leftover token")),
})
}
}
Err(InternalAstError::NoMatch) => Err(AstError::UnexpectedToken {
token: (*state.peek()).to_owned(),
additional: None,
}),
Err(InternalAstError::UnexpectedToken { token, additional }) => {
Err(AstError::UnexpectedToken {
token: (*token).to_owned(),
additional: additional.map(Cow::Borrowed),
})
}
}
}
}
pub fn nodes(&self) -> &Block<'a> {
&self.nodes
}
pub fn nodes_mut(&mut self) -> &mut Block<'a> {
&mut self.nodes
}
pub fn iter_tokens(&self) -> impl Iterator<Item = &Token<'a>> {
self.tokens.iter().map(|(_, token)| token).sorted()
}
pub fn update_positions(&mut self) {
use crate::tokenizer::Position;
let mut start_position = Position {
bytes: 0,
character: 1,
line: 1,
};
let mut next_is_new_line = false;
for (_, token) in self.tokens.iter() {
let display = token.to_string();
let new_lines = match bytecount::count(display.as_bytes(), b'\n') {
0 | 1 => 0,
n => n,
};
let end_position = if token.token_kind() == TokenKind::Eof {
start_position
} else {
let mut end_position = Position {
bytes: start_position.bytes() + display.len(),
line: start_position.line() + new_lines,
character: {
let offset = display.lines().last().unwrap_or("").chars().count();
if new_lines > 0 || next_is_new_line {
offset + 1
} else {
start_position.character() + offset
}
},
};
if next_is_new_line {
end_position.line += 1;
next_is_new_line = false;
}
end_position
};
if display.ends_with('\n') {
next_is_new_line = true;
}
token.start_position.store(start_position);
token.end_position.store(end_position);
start_position = end_position;
}
}
}