use crate::parser::context::ParserCtx;
use crate::parser::diagnostics::{DiagnosticKind, ParseDiagnostic, push_diagnostic};
use crate::parser::events::{Event, ExprParse, finish, push_range};
use crate::parser::lexer::{TokKind, Token, is_op_suffix_char};
use crate::parser::recovery::{error_expr_to_line_end, error_expr_with_range};
use crate::parser::structural::{
KwOperandContext, KwStmt, is_op_name, parse_abstract_type, parse_begin_expr, parse_do_block,
parse_for_expr, parse_function_expr, parse_if_expr, parse_import_stmt, parse_keyword_stmt,
parse_let_expr, parse_macro_def, parse_module_expr, parse_name_list_stmt, parse_primitive_type,
parse_quote_expr, parse_struct_expr, parse_try_expr, parse_typegroup_expr, parse_while_expr,
};
use crate::syntax::SyntaxKind;
mod array;
mod juxtapose;
mod macros;
mod prec;
pub(crate) use array::parse_for_specs;
use array::{
BRACE_CAT, BRACKET_CAT, array_element_boundary, parse_comprehension, parse_delimited_literal,
parse_element, parse_empty_ncat, parse_generator_clauses, parse_matrix,
};
use juxtapose::{
is_element_of_tok, is_for_separator_tok, lhs_is_number, should_juxtapose,
should_juxtapose_string_error, signed_literal_fold,
};
pub(crate) use macros::{is_var_identifier_start, push_var_macro_name};
use macros::{parse_macro, parse_qualified_macro};
pub(crate) use prec::is_comparison_op as token_is_comparison_op;
use prec::{
COMMA_BP, COMMA_ITEM_BP, JUXTAPOSE_L, JUXTAPOSE_R, PREFIX_BP, SPLAT_BP, TERNARY_BRANCH_BP,
TERNARY_L, WHERE_BOUND_BP, WHERE_BP, WORD_OP_L, WORD_OP_R, infix_binding_power,
is_assignment_op, is_comparison_op, is_flat_arith_op, is_lone_error_operator, is_operator,
is_quotable_operator, is_unary_prefix_op, is_value_operator, next_operator, word_operator,
};
#[derive(Clone, Copy, Default)]
struct ExprFlags {
inside_brackets: bool,
no_range: bool,
array_mode: bool,
generator_for_ends: bool,
paren_content_start: Option<usize>,
end_marker: bool,
begin_marker: bool,
public_context: bool,
stmt_comma: bool,
for_spec_var: bool,
no_where: bool,
no_juxtapose: bool,
no_decl_where: bool,
name_context: bool,
field_access_rhs: bool,
kw_generator_body: bool,
}
impl ExprFlags {
fn operator_newlines(self) -> bool {
self.inside_brackets && !self.array_mode
}
}
fn newline_stops_operand(ctx: &ParserCtx<'_>, op_idx: usize, flags: ExprFlags) -> bool {
let newline_significant = !flags.inside_brackets || flags.array_mode;
newline_significant
&& ctx
.token(ctx.skip_ws_and_comments(op_idx + 1))
.map(|t| t.kind)
== Some(TokKind::Newline)
}
pub(crate) fn parse_expr(
tokens: &ParserCtx<'_>,
start: usize,
min_bp: u8,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
parse_expr_in(tokens, start, min_bp, diagnostics, ExprFlags::default())
}
pub(crate) fn parse_stmt(
tokens: &ParserCtx<'_>,
start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
parse_block_stmt(tokens, start, true, diagnostics)
}
pub(crate) fn parse_block_stmt(
tokens: &ParserCtx<'_>,
start: usize,
public_context: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
public_context,
stmt_comma: true,
..ExprFlags::default()
};
parse_expr_in(tokens, start, 0, diagnostics, flags)
}
pub(crate) fn parse_kw_stmt_operand(
tokens: &ParserCtx<'_>,
start: usize,
context: KwOperandContext,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
stmt_comma: true,
kw_generator_body: context.for_ends,
end_marker: context.end_marker,
begin_marker: context.begin_marker,
no_range: context.colon_ends,
..ExprFlags::default()
};
parse_expr_in(tokens, start, 0, diagnostics, flags)
}
pub(crate) fn parse_break_value(
tokens: &ParserCtx<'_>,
start: usize,
colon_ends: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
stmt_comma: true,
no_range: colon_ends,
..ExprFlags::default()
};
parse_expr_in(tokens, start, 0, diagnostics, flags)
}
pub(crate) fn parse_expr_in_brackets(
tokens: &ParserCtx<'_>,
start: usize,
min_bp: u8,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
inside_brackets: true,
generator_for_ends: true,
end_marker,
begin_marker: end_marker,
..ExprFlags::default()
};
parse_expr_in(tokens, start, min_bp, diagnostics, flags)
}
pub(crate) fn parse_signature_expr(
tokens: &ParserCtx<'_>,
start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
no_decl_where: true,
name_context: true,
..ExprFlags::default()
};
parse_expr_in(tokens, start, 0, diagnostics, flags)
}
pub(crate) fn parse_type_spec_expr(
tokens: &ParserCtx<'_>,
start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
array_mode: true,
name_context: true,
..ExprFlags::default()
};
parse_expr_in(tokens, start, 0, diagnostics, flags)
}
pub(crate) fn parse_name_signature_expr(
tokens: &ParserCtx<'_>,
start: usize,
min_bp: u8,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
name_context: true,
..ExprFlags::default()
};
parse_expr_in(tokens, start, min_bp, diagnostics, flags)
}
fn parse_expr_in(
tokens: &ParserCtx<'_>,
start: usize,
min_bp: u8,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
) -> Option<ExprParse> {
let ExprFlags {
inside_brackets: _,
no_range,
array_mode,
generator_for_ends: _,
paren_content_start: _,
end_marker: _,
begin_marker,
public_context,
stmt_comma,
for_spec_var,
no_where,
no_juxtapose,
no_decl_where,
name_context,
field_access_rhs: _,
kw_generator_body,
} = flags;
let kw_context = KwOperandContext {
for_ends: !stmt_comma || kw_generator_body,
..KwOperandContext::default()
};
let ctx = tokens;
if public_context && is_public_keyword(ctx, start) {
return parse_name_list_stmt(tokens, start, SyntaxKind::PUBLIC_STMT, diagnostics);
}
let name_error_atom = (name_context
&& (ctx
.token(start)
.is_some_and(|t| is_name_error_keyword(t.kind))
|| is_reserved_typegroup(ctx, start)))
.then(|| {
let pos = ctx.token(start).map_or(start, |t| t.start);
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidNameKeyword,
"reserved keyword used as a name",
pos,
ctx.token(start).map_or(pos, |t| t.end),
);
keyword_name_error_atom(start)
});
let mut block_form = if name_error_atom.is_some() {
None
} else if let Some(decl_word) = type_decl_keyword(ctx, start) {
Some(match decl_word {
TypeDecl::Abstract => parse_abstract_type(tokens, start, diagnostics),
TypeDecl::Primitive => parse_primitive_type(tokens, start, diagnostics),
})
} else if opens_typegroup(ctx, start) {
Some(parse_typegroup_expr(tokens, start, diagnostics))
} else {
match ctx.token(start).map(|t| t.kind) {
Some(TokKind::IfKw) => Some(parse_if_expr(tokens, start, diagnostics)),
Some(TokKind::FunctionKw) => Some(parse_function_expr(tokens, start, diagnostics)),
Some(TokKind::MacroKw) => Some(parse_macro_def(tokens, start, diagnostics)),
Some(TokKind::BeginKw) if !begin_marker => {
Some(parse_begin_expr(tokens, start, diagnostics))
}
Some(TokKind::QuoteKw) => Some(parse_quote_expr(tokens, start, diagnostics)),
Some(TokKind::WhileKw) => Some(parse_while_expr(tokens, start, diagnostics)),
Some(TokKind::ForKw) => Some(parse_for_expr(tokens, start, diagnostics)),
Some(TokKind::LetKw) => Some(parse_let_expr(tokens, start, diagnostics)),
Some(TokKind::TryKw) => Some(parse_try_expr(tokens, start, diagnostics)),
Some(TokKind::StructKw | TokKind::MutableKw) => {
Some(parse_struct_expr(tokens, start, diagnostics))
}
Some(TokKind::ModuleKw | TokKind::BaremoduleKw) => {
Some(parse_module_expr(tokens, start, diagnostics))
}
_ => None,
}
};
if block_form.is_none() {
match ctx.token(start).map(|t| t.kind) {
Some(TokKind::ReturnKw) => {
let parsed = parse_keyword_stmt(
tokens,
start,
SyntaxKind::RETURN_EXPR,
KwStmt::ExprTuple {
optional_value: true,
context: KwOperandContext {
end_marker: flags.end_marker
&& ctx.options.syntax_edition
>= crate::parser::SyntaxEdition::Julia114,
begin_marker: begin_marker
&& ctx.options.syntax_edition
>= crate::parser::SyntaxEdition::Julia114,
colon_ends: no_range
&& ctx.options.syntax_edition
>= crate::parser::SyntaxEdition::Julia114,
..kw_context
},
},
diagnostics,
)?;
if name_error_atom.is_none()
&& stmt_comma
&& min_bp <= COMMA_BP
&& ctx
.token(ctx.skip_ws_and_block_comments(parsed.end))
.is_some_and(|token| token.kind == TokKind::Comma)
{
block_form = Some(Some(parsed));
} else {
return Some(parsed);
}
}
Some(kind @ (TokKind::BreakKw | TokKind::ContinueKw)) => {
let is_break = kind == TokKind::BreakKw;
let parsed = parse_keyword_stmt(
tokens,
start,
if is_break {
SyntaxKind::BREAK_EXPR
} else {
SyntaxKind::CONTINUE_EXPR
},
KwStmt::Label {
takes_value: is_break,
colon_ends: no_range,
},
diagnostics,
)?;
let following = ctx
.token(ctx.skip_ws_and_block_comments(parsed.end))
.map(|t| t.kind);
let range = following == Some(TokKind::Colon)
&& ctx.options.syntax_edition == crate::parser::SyntaxEdition::Julia113
&& !no_range;
if name_error_atom.is_some() || !(range || following == Some(TokKind::Comma)) {
return Some(parsed);
}
block_form = Some(Some(parsed));
}
Some(TokKind::ConstKw) => {
return parse_keyword_stmt(
tokens,
start,
SyntaxKind::CONST_STMT,
KwStmt::ExprTuple {
optional_value: false,
context: kw_context,
},
diagnostics,
);
}
Some(TokKind::GlobalKw) => {
return parse_keyword_stmt(
tokens,
start,
SyntaxKind::GLOBAL_STMT,
KwStmt::ExprTuple {
optional_value: false,
context: kw_context,
},
diagnostics,
);
}
Some(TokKind::LocalKw) => {
return parse_keyword_stmt(
tokens,
start,
SyntaxKind::LOCAL_STMT,
KwStmt::ExprTuple {
optional_value: false,
context: kw_context,
},
diagnostics,
);
}
Some(TokKind::ImportKw) => {
return parse_import_stmt(tokens, start, SyntaxKind::IMPORT_STMT, diagnostics);
}
Some(TokKind::UsingKw) => {
return parse_import_stmt(tokens, start, SyntaxKind::USING_STMT, diagnostics);
}
Some(TokKind::ExportKw) => {
return parse_name_list_stmt(tokens, start, SyntaxKind::EXPORT_STMT, diagnostics);
}
_ => {}
}
}
let (mut lhs, mut lhs_is_block_keyword) = match (name_error_atom, block_form) {
(Some(atom), _) => (atom, false),
(None, Some(parsed)) => (parsed?, true),
(None, None)
if stmt_comma
&& min_bp <= COMMA_BP
&& ctx
.token(start)
.is_some_and(|token| token.kind == TokKind::Comma) =>
{
let comma = &tokens[start];
push_diagnostic(
diagnostics,
DiagnosticKind::EmptyListSlot,
"unexpected `,` at the start of a statement",
comma.start,
comma.start,
);
let missing = ExprParse {
start,
end: start,
events: Vec::new(),
};
(
parse_comma_tuple(tokens, ctx, missing, diagnostics, flags),
false,
)
}
(None, None) => (parse_prefix(ctx, start, diagnostics, flags)?, false),
};
let mut glued_colon_done = false;
loop {
if !lhs_is_block_keyword {
lhs = parse_postfix_chain(ctx, lhs, array_mode, flags.end_marker, diagnostics);
}
if !lhs_is_block_keyword
&& !no_juxtapose
&& should_juxtapose_string_error(ctx, &lhs, min_bp)
&& let Some(rhs) = parse_expr_in(tokens, lhs.end, JUXTAPOSE_R, diagnostics, flags)
{
let pos = tokens[lhs.end - 1].end;
push_diagnostic(
diagnostics,
DiagnosticKind::StringJuxtapose,
"invalid juxtaposition",
pos,
pos,
);
lhs = build_binary(SyntaxKind::JUXTAPOSE_EXPR, lhs, rhs);
continue;
}
if !lhs_is_block_keyword
&& !no_juxtapose
&& should_juxtapose(ctx, &lhs, min_bp)
&& let Some(rhs) = parse_expr_in(
tokens,
lhs.end,
JUXTAPOSE_R,
diagnostics,
ExprFlags {
no_juxtapose: true,
..flags
},
)
{
if matches!(
lhs.events.first(),
Some(Event::Start(SyntaxKind::JUXTAPOSE_EXPR))
) {
lhs.events.pop();
push_range(&mut lhs.events, lhs.end, rhs.start);
lhs.events.extend(rhs.events);
lhs.events.push(Event::Finish);
lhs.end = rhs.end;
} else {
lhs = build_binary(SyntaxKind::JUXTAPOSE_EXPR, lhs, rhs);
}
continue;
}
lhs_is_block_keyword = false;
if SPLAT_BP >= min_bp {
let splat_idx = if flags.operator_newlines() {
ctx.skip_trivia(lhs.end)
} else {
ctx.skip_ws(lhs.end)
};
if ctx.token(splat_idx).map(|t| t.kind) == Some(TokKind::DotDotDot) {
let mut events = vec![Event::Start(SyntaxKind::SPLAT_EXPR)];
events.extend(lhs.events);
push_range(&mut events, lhs.end, splat_idx);
events.push(Event::Tok(splat_idx));
events.push(Event::Finish);
lhs = ExprParse {
start: lhs.start,
end: splat_idx + 1,
events,
};
continue;
}
}
if !no_where
&& WHERE_BP >= min_bp
&& let Some((where_idx, TokKind::WhereKw)) =
next_operator(ctx, lhs.end, flags.operator_newlines())
{
lhs = parse_where_chain(tokens, ctx, lhs, where_idx, diagnostics, flags);
continue;
}
if stmt_comma
&& min_bp <= COMMA_BP
&& ctx
.token(ctx.skip_ws_and_block_comments(lhs.end))
.map(|t| t.kind)
== Some(TokKind::Comma)
{
lhs = parse_comma_tuple(tokens, ctx, lhs, diagnostics, flags);
continue;
}
if !for_spec_var
&& let Some(op_idx) = word_operator(ctx, lhs.end, flags.operator_newlines())
{
if WORD_OP_L < min_bp {
break;
}
let rhs_operand = ctx.skip_trivia(op_idx + 1);
let Some(rhs) = parse_expr_in(tokens, rhs_operand, WORD_OP_R, diagnostics, flags)
else {
let op = &tokens[op_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingOperand,
"expected right-hand side for operator",
op.start,
op.end,
);
return Some(error_expr_to_line_end(tokens, lhs.start, op_idx + 1));
};
lhs = build_binary(SyntaxKind::BINARY_EXPR, lhs, rhs);
continue;
}
let Some((op_idx, op_kind)) = next_operator(ctx, lhs.end, flags.operator_newlines()) else {
break;
};
if for_spec_var && is_element_of_tok(&tokens[op_idx]) {
break;
}
if array_mode && array_element_boundary(ctx, lhs.end, op_idx) {
break;
}
if op_kind == TokKind::Dot
&& ctx.token(ctx.skip_trivia(op_idx + 1)).map(|t| t.kind) == Some(TokKind::At)
{
lhs = parse_qualified_macro(ctx, lhs, op_idx, diagnostics, flags);
continue;
}
if no_range && op_kind == TokKind::Colon {
break;
}
if op_kind == TokKind::Colon
&& matches!(
ctx.token(op_idx + 1).map(|t| t.kind),
Some(TokKind::Lt | TokKind::Gt)
)
{
if glued_colon_done {
break;
}
let (l_bp, r_bp) = infix_binding_power(TokKind::Colon).expect("colon binds");
if l_bp < min_bp {
break;
}
let colon = &tokens[op_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidGluedOperator,
"invalid operator",
colon.start,
colon.start,
);
let lt_idx = op_idx + 1;
let rhs_operand = ctx.skip_trivia(lt_idx + 1);
lhs = match parse_expr_in(tokens, rhs_operand, r_bp, diagnostics, flags) {
Some(rhs) => build_binary(SyntaxKind::BINARY_EXPR, lhs, rhs),
None => {
let lt = &tokens[lt_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingOperand,
"expected right-hand side for operator",
colon.start,
lt.end,
);
build_binary_missing_rhs(SyntaxKind::BINARY_EXPR, lhs, lt_idx + 1)
}
};
glued_colon_done = true;
continue;
}
if op_kind == TokKind::Colon {
if glued_colon_done {
break;
}
let (l_bp, _) = infix_binding_power(TokKind::Colon).expect("colon binds");
if l_bp < min_bp {
break;
}
lhs = parse_colon_range(tokens, ctx, lhs, op_idx, diagnostics, flags);
continue;
}
if is_comparison_op(op_kind) {
let (l_bp, _) = infix_binding_power(op_kind).expect("comparison binds");
if l_bp < min_bp {
break;
}
lhs = parse_operator_chain(
ctx,
lhs,
op_idx,
diagnostics,
flags,
ChainSpec {
single: operator_node_kind(op_kind),
flat: SyntaxKind::COMPARISON_EXPR,
continues: |kind, _| is_comparison_op(kind),
},
);
continue;
}
if is_flat_arith_op(&tokens[op_idx]) {
let (l_bp, _) = infix_binding_power(op_kind).expect("arith binds");
if l_bp < min_bp {
break;
}
lhs = parse_operator_chain(
ctx,
lhs,
op_idx,
diagnostics,
flags,
ChainSpec {
single: SyntaxKind::BINARY_EXPR,
flat: SyntaxKind::BINARY_EXPR,
continues: |kind, idx| kind == op_kind && is_flat_arith_op(&tokens[idx]),
},
);
continue;
}
if op_kind == TokKind::Question {
if TERNARY_L < min_bp {
break;
}
lhs = match parse_ternary(ctx, lhs, op_idx, diagnostics, flags) {
Ok(node) => node,
Err(done) => return Some(done),
};
continue;
}
let (l_bp, r_bp) = if is_assignment_op(op_kind) {
(2, 1)
} else {
match infix_binding_power(op_kind) {
Some(bp) => bp,
None => break,
}
};
if l_bp < min_bp {
break;
}
if op_kind == TokKind::Arrow {
lhs = reclassify_arrow_parameters(lhs);
}
let rhs_operand = ctx.skip_trivia(op_idx + 1);
let rhs_result = if op_kind == TokKind::Dot {
parse_prefix(
ctx,
rhs_operand,
diagnostics,
ExprFlags {
field_access_rhs: true,
..flags
},
)
} else {
parse_expr_in(
tokens,
rhs_operand,
r_bp,
diagnostics,
ExprFlags {
no_juxtapose: false,
..flags
},
)
};
let Some(mut rhs) = rhs_result else {
let op = &tokens[op_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingOperand,
"expected right-hand side for operator",
op.start,
op.end,
);
lhs = build_binary_missing_rhs(operator_node_kind(op_kind), lhs, rhs_operand);
continue;
};
if op_kind == TokKind::ColonColon
&& !no_where
&& !no_decl_where
&& let Some((where_idx, TokKind::WhereKw)) =
next_operator(ctx, rhs.end, flags.operator_newlines())
{
rhs = parse_where_chain(tokens, ctx, rhs, where_idx, diagnostics, flags);
}
let node = operator_node_kind(op_kind);
if op_kind == TokKind::Dot && op_idx > lhs.end {
let op = &tokens[op_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::DotWhitespace,
"whitespace before `.`",
op.end,
op.end,
);
}
lhs = build_binary(node, lhs, rhs);
}
Some(lhs)
}
fn parse_where_chain(
tokens: &ParserCtx<'_>,
ctx: &ParserCtx<'_>,
mut lhs: ExprParse,
mut where_idx: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
) -> ExprParse {
let bound_flags = ExprFlags {
no_where: true,
..flags
};
loop {
let bound_start = ctx.skip_trivia(where_idx + 1);
let Some(bound) = parse_expr_in(
tokens,
bound_start,
WHERE_BOUND_BP,
diagnostics,
bound_flags,
) else {
let op = &tokens[where_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingWhereBound,
"expected type bound after `where`",
op.start,
op.end,
);
return error_expr_to_line_end(tokens, lhs.start, where_idx + 1);
};
lhs = build_binary(SyntaxKind::WHERE_EXPR, lhs, bound);
match next_operator(ctx, lhs.end, flags.operator_newlines()) {
Some((idx, TokKind::WhereKw)) => where_idx = idx,
_ => return lhs,
}
}
}
enum TypeDecl {
Abstract,
Primitive,
}
fn type_decl_keyword(ctx: &ParserCtx<'_>, start: usize) -> Option<TypeDecl> {
let word = match ctx.token(start) {
Some(t) if t.kind == TokKind::Ident && t.text == "abstract" => TypeDecl::Abstract,
Some(t) if t.kind == TokKind::Ident && t.text == "primitive" => TypeDecl::Primitive,
_ => return None,
};
let next = ctx.token(ctx.skip_trivia(start + 1))?;
(next.kind == TokKind::Ident && next.text == "type").then_some(word)
}
fn is_typegroup(ctx: &ParserCtx<'_>, start: usize) -> bool {
ctx.token(start)
.is_some_and(|t| t.kind == TokKind::Ident && t.text == "typegroup")
}
fn is_reserved_typegroup(ctx: &ParserCtx<'_>, start: usize) -> bool {
ctx.options.syntax_edition >= crate::parser::SyntaxEdition::Julia114 && is_typegroup(ctx, start)
}
fn opens_typegroup(ctx: &ParserCtx<'_>, start: usize) -> bool {
if !is_typegroup(ctx, start) {
return false;
}
if ctx.options.syntax_edition >= crate::parser::SyntaxEdition::Julia114 {
return true;
}
ctx.token(ctx.skip_ws_and_block_comments(start + 1))
.is_some_and(|token| match token.kind {
TokKind::StructKw | TokKind::MutableKw | TokKind::At | TokKind::StringDelimOpen => true,
TokKind::Ident => matches!(token.text, "abstract" | "primitive"),
_ => false,
})
}
fn is_public_keyword(ctx: &ParserCtx<'_>, start: usize) -> bool {
match ctx.token(start) {
Some(t) if t.kind == TokKind::Ident && t.text == "public" => {}
_ => return false,
}
match ctx.token(ctx.skip_trivia(start + 1)).map(|t| t.kind) {
Some(TokKind::LParen | TokKind::Eq | TokKind::LBracket) | None => false,
Some(_) => true,
}
}
fn build_operands(kind: SyntaxKind, operands: Vec<ExprParse>, tail: Option<usize>) -> ExprParse {
let mut iter = operands.into_iter();
let first = iter.next().expect("node needs at least one operand");
let start = first.start;
let mut end = first.end;
let mut events = vec![Event::Start(kind)];
events.extend(first.events);
for operand in iter {
push_range(&mut events, end, operand.start);
events.extend(operand.events);
end = operand.end;
}
if let Some(gap_end) = tail {
push_range(&mut events, end, gap_end);
end = gap_end;
}
events.push(Event::Finish);
ExprParse { start, end, events }
}
fn build_binary(kind: SyntaxKind, lhs: ExprParse, rhs: ExprParse) -> ExprParse {
build_operands(kind, vec![lhs, rhs], None)
}
fn operator_node_kind(op_kind: TokKind) -> SyntaxKind {
match op_kind {
k if is_assignment_op(k) => SyntaxKind::ASSIGNMENT_EXPR,
TokKind::Arrow => SyntaxKind::ARROW_EXPR,
TokKind::ColonColon => SyntaxKind::TYPE_ANNOTATION,
_ => SyntaxKind::BINARY_EXPR,
}
}
fn reclassify_arrow_parameters(mut lhs: ExprParse) -> ExprParse {
let is_parenthesized = matches!(
lhs.events.first(),
Some(Event::Start(
SyntaxKind::PAREN_EXPR | SyntaxKind::PAREN_BLOCK
))
);
let wraps_where = lhs
.events
.iter()
.filter_map(|event| match event {
Event::Start(kind) => Some(*kind),
Event::Tok(_) | Event::Finish => None,
})
.skip(1)
.find(|kind| *kind != SyntaxKind::PAREN_EXPR)
== Some(SyntaxKind::WHERE_EXPR);
if is_parenthesized && !wraps_where {
lhs.events[0] = Event::Start(SyntaxKind::TUPLE_EXPR);
}
lhs
}
fn build_binary_missing_rhs(kind: SyntaxKind, lhs: ExprParse, gap_end: usize) -> ExprParse {
build_operands(kind, vec![lhs], Some(gap_end))
}
fn parse_comma_tuple(
tokens: &ParserCtx<'_>,
ctx: &ParserCtx<'_>,
first: ExprParse,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
) -> ExprParse {
let start = first.start;
let mut events = vec![Event::Start(SyntaxKind::BARE_TUPLE_EXPR)];
let mut end = first.end;
events.extend(first.events);
loop {
let comma_idx = ctx.skip_ws_and_block_comments(end);
if ctx.token(comma_idx).map(|t| t.kind) != Some(TokKind::Comma) {
break;
}
push_range(&mut events, end, comma_idx);
events.push(Event::Tok(comma_idx));
end = comma_idx + 1;
let item_start = ctx.skip_trivia(end);
if ctx
.token(item_start)
.is_some_and(|t| is_lone_error_operator(t.kind))
{
break;
}
let item = if comma_idx == start
&& let Some(token) = ctx.token(item_start)
&& is_closing_block_keyword(token.kind)
&& !(flags.end_marker && token.kind == TokKind::EndKw)
{
push_diagnostic(
diagnostics,
DiagnosticKind::StrayKeyword,
"unexpected block keyword",
token.start,
token.end,
);
Some(error_expr_with_range(item_start, item_start + 1))
} else {
parse_expr_in(tokens, item_start, COMMA_ITEM_BP, diagnostics, flags)
};
match item {
Some(item) => {
push_range(&mut events, end, item.start);
events.extend(item.events);
end = item.end;
}
None => break,
}
}
events.push(Event::Finish);
ExprParse { start, end, events }
}
fn parse_colon_range(
tokens: &ParserCtx<'_>,
ctx: &ParserCtx<'_>,
lhs: ExprParse,
first_colon: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
) -> ExprParse {
let (_, r_bp) = infix_binding_power(TokKind::Colon).expect("colon binds");
let mut head = lhs;
let mut step: Option<ExprParse> = None;
let mut op_idx = first_colon;
loop {
let newline_stop = newline_stops_operand(ctx, op_idx, flags);
let rhs_operand = ctx.skip_trivia(op_idx + 1);
let rhs = if newline_stop {
None
} else {
parse_expr_in(tokens, rhs_operand, r_bp, diagnostics, flags)
};
let Some(rhs) = rhs else {
let op = &tokens[op_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingOperand,
"expected right-hand side for operator",
op.start,
op.end,
);
return match step.take() {
None => build_binary_missing_rhs(SyntaxKind::BINARY_EXPR, head, op_idx + 1),
Some(mid) => {
build_operands(SyntaxKind::RANGE_EXPR, vec![head, mid], Some(op_idx + 1))
}
};
};
let last_end = rhs.end;
match step.take() {
Some(mid) => head = build_operands(SyntaxKind::RANGE_EXPR, vec![head, mid, rhs], None),
None => step = Some(rhs),
}
let continues = match next_operator(ctx, last_end, flags.operator_newlines()) {
Some((idx, TokKind::Colon)) if !flags.no_range => {
let split = flags.array_mode && array_element_boundary(ctx, last_end, idx);
(!split).then_some(idx)
}
_ => None,
};
match continues {
Some(idx) => op_idx = idx,
None => break,
}
}
match step {
Some(mid) => build_binary(SyntaxKind::BINARY_EXPR, head, mid),
None => head,
}
}
struct ChainSpec<F: Fn(TokKind, usize) -> bool> {
single: SyntaxKind,
flat: SyntaxKind,
continues: F,
}
fn parse_operator_chain(
ctx: &ParserCtx<'_>,
lhs: ExprParse,
first_op: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
spec: ChainSpec<impl Fn(TokKind, usize) -> bool>,
) -> ExprParse {
let tokens = ctx.tokens();
let mut operands = vec![lhs];
let mut op_count = 0usize;
let mut op_idx = first_op;
let tail = loop {
let (_, r_bp) = infix_binding_power(tokens[op_idx].kind).expect("chain operator binds");
let rhs_operand = ctx.skip_trivia(op_idx + 1);
let Some(rhs) = parse_expr_in(tokens, rhs_operand, r_bp, diagnostics, flags) else {
let op = &tokens[op_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingOperand,
"expected right-hand side for operator",
op.start,
op.end,
);
op_count += 1;
break Some(op_idx + 1);
};
let last_end = rhs.end;
operands.push(rhs);
op_count += 1;
let continues = match next_operator(ctx, last_end, flags.operator_newlines()) {
Some((idx, kind)) if (spec.continues)(kind, idx) => {
let split = flags.array_mode && array_element_boundary(ctx, last_end, idx);
(!split).then_some(idx)
}
_ => None,
};
match continues {
Some(idx) => op_idx = idx,
None => break None,
}
};
let kind = if op_count == 1 {
spec.single
} else {
spec.flat
};
build_operands(kind, operands, tail)
}
fn parse_prefix(
ctx: &ParserCtx<'_>,
start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
) -> Option<ExprParse> {
let tok = ctx.token(start)?;
match tok.kind {
k if is_curly_operator_name(k)
&& ctx.token(start + 1).map(|t| t.kind) == Some(TokKind::LBrace) =>
{
Some(ExprParse {
start,
end: start + 1,
events: vec![Event::Tok(start)],
})
}
k if flags.field_access_rhs && k.is_keyword() => Some(atom(SyntaxKind::NAME, start)),
TokKind::EndKw if flags.end_marker => Some(atom(SyntaxKind::END_MARKER, start)),
TokKind::BeginKw if flags.begin_marker => Some(atom(SyntaxKind::BEGIN_MARKER, start)),
k if flags.name_context
&& k != TokKind::Colon
&& is_value_operator(k)
&& !(k == TokKind::Amp
&& ctx.token(start + 1).map(|t| t.kind) == Some(TokKind::LParen)) =>
{
if ctx.token(start + 1).map(|t| t.kind) == Some(TokKind::LParen) {
let (list_events, end) = parse_arg_list(
ctx,
start + 1,
TokKind::RParen,
SyntaxKind::ARG_LIST,
flags.end_marker,
true,
diagnostics,
);
let mut events = vec![Event::Start(SyntaxKind::CALL_EXPR), Event::Tok(start)];
events.extend(list_events);
events.push(Event::Finish);
Some(ExprParse { start, end, events })
} else {
Some(atom(SyntaxKind::OPERATOR_ATOM, start))
}
}
k if matches!(k, TokKind::Plus | TokKind::Minus) && signed_literal_fold(ctx, start) => {
let num = start + 1;
Some(ExprParse {
start,
end: num + 1,
events: vec![
Event::Start(SyntaxKind::LITERAL),
Event::Tok(start),
Event::Tok(num),
Event::Finish,
],
})
}
k if is_unary_prefix_op(k, tok.text) => {
let op_suffixed = matches!(
tok.kind,
TokKind::Plus
| TokKind::Minus
| TokKind::DotPlus
| TokKind::DotMinus
| TokKind::DotPlusPercent
| TokKind::DotMinusPercent
) && tok.text.chars().any(is_op_suffix_char);
let is_unary_paren_op = matches!(
tok.kind,
TokKind::Plus
| TokKind::Minus
| TokKind::PlusPercent
| TokKind::MinusPercent
| TokKind::DotPlus
| TokKind::DotMinus
| TokKind::DotPlusPercent
| TokKind::DotMinusPercent
| TokKind::Bang
| TokKind::Tilde
| TokKind::DotTilde
| TokKind::Subtype
| TokKind::Supertype
| TokKind::UniRadical
| TokKind::UniPlus
| TokKind::UniTimes
);
let paren_idx = ctx.skip_ws(start + 1);
let spaced = paren_idx > start + 1;
let next_is_lparen = ctx.token(paren_idx).map(|t| t.kind) == Some(TokKind::LParen);
let glued_call = !spaced && (op_suffixed || unary_op_paren_is_call(ctx, paren_idx));
let spaced_call = spaced && !op_suffixed && unary_op_paren_is_call(ctx, paren_idx);
if is_unary_paren_op && next_is_lparen && (glued_call || spaced_call) {
if spaced_call {
let opener = &ctx.tokens()[paren_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::PrefixOpenerWhitespace,
"whitespace before opener",
opener.start,
opener.start,
);
}
let (list_events, end) = parse_arg_list(
ctx,
paren_idx,
TokKind::RParen,
SyntaxKind::ARG_LIST,
flags.end_marker,
true,
diagnostics,
);
let mut events = vec![Event::Start(SyntaxKind::CALL_EXPR), Event::Tok(start)];
push_range(&mut events, start + 1, paren_idx);
events.extend(list_events);
events.push(Event::Finish);
return Some(ExprParse { start, end, events });
}
let node = if tok.kind == TokKind::ColonColon {
SyntaxKind::TYPE_ANNOTATION
} else {
SyntaxKind::UNARY_EXPR
};
let operand_start = ctx.skip_trivia(start + 1);
if ctx.token(operand_start).map(|t| t.kind) == Some(TokKind::Eq)
&& !matches!(tok.kind, TokKind::Amp | TokKind::ColonColon)
{
return Some(atom(SyntaxKind::OPERATOR_ATOM, start));
}
let newline_stop = newline_stops_operand(ctx, start, flags);
if newline_stop && tok.kind != TokKind::ColonColon {
return Some(atom(SyntaxKind::OPERATOR_ATOM, start));
}
let is_subtype = matches!(tok.kind, TokKind::Subtype | TokKind::Supertype);
let operand_flags = ExprFlags {
no_range: false,
array_mode: flags.array_mode,
no_where: !is_subtype || flags.no_where,
no_juxtapose: !is_subtype,
..flags
};
let operand_bp = if is_subtype { WHERE_BP } else { PREFIX_BP };
let Some(operand) = parse_expr_in(
ctx.tokens(),
operand_start,
operand_bp,
diagnostics,
operand_flags,
) else {
if tok.kind == TokKind::ColonColon {
return Some(error_expr_with_range(start, start + 1));
}
return Some(atom(SyntaxKind::OPERATOR_ATOM, start));
};
if op_suffixed {
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidPrefixOperator,
"invalid operator in prefix position",
tok.start,
tok.end,
);
let mut events = vec![Event::Start(SyntaxKind::UNARY_EXPR)];
events.extend(error_wrapped_atom(SyntaxKind::OPERATOR_ATOM, start).events);
push_range(&mut events, start + 1, operand.start);
events.extend(operand.events);
finish(&mut events, SyntaxKind::UNARY_EXPR);
return Some(ExprParse {
start,
end: operand.end,
events,
});
}
let mut events = vec![Event::Start(node)];
push_range(&mut events, start, operand.start);
events.extend(operand.events);
events.push(Event::Finish);
Some(ExprParse {
start,
end: operand.end,
events,
})
}
k if is_operator_call_name(k) && operator_call_lparen(ctx, start, flags).is_some() => {
let paren_idx = operator_call_lparen(ctx, start, flags).unwrap();
if paren_idx > start + 1 {
let opener = &ctx.tokens()[paren_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::OpenerWhitespace,
"whitespace before opener",
opener.start,
opener.start,
);
}
let (list_events, end) = parse_arg_list(
ctx,
paren_idx,
TokKind::RParen,
SyntaxKind::ARG_LIST,
flags.end_marker,
true,
diagnostics,
);
let mut events = vec![Event::Start(SyntaxKind::CALL_EXPR), Event::Tok(start)];
push_range(&mut events, start + 1, paren_idx);
events.extend(list_events);
events.push(Event::Finish);
Some(ExprParse { start, end, events })
}
TokKind::Colon => parse_quote_sym(
ctx,
start,
diagnostics,
!flags.field_access_rhs,
flags.end_marker,
flags.inside_brackets,
flags.generator_for_ends,
)
.or_else(|| Some(atom(SyntaxKind::OPERATOR_ATOM, start))),
TokKind::Dollar => Some(parse_prefix_interpolation(ctx, start, diagnostics)),
TokKind::At => Some(parse_macro(ctx, start, diagnostics, flags)),
TokKind::LParen => parse_paren(
ctx,
start,
flags.end_marker,
flags.generator_for_ends,
diagnostics,
),
TokKind::LBracket => Some(parse_delimited_literal(
ctx,
start,
flags.end_marker,
BRACKET_CAT,
diagnostics,
)),
TokKind::LBrace => Some(parse_delimited_literal(
ctx,
start,
flags.end_marker,
BRACE_CAT,
diagnostics,
)),
TokKind::Ident => Some(atom(SyntaxKind::NAME, start)),
TokKind::WhereKw => Some(atom(SyntaxKind::NAME, start)),
TokKind::StringPrefix | TokKind::StringDelimOpen | TokKind::CmdDelimOpen => {
Some(parse_string_literal(ctx, start, diagnostics))
}
TokKind::Char => {
let tok = &ctx.tokens()[start];
if !char_token_terminated(tok.text) {
push_diagnostic(
diagnostics,
DiagnosticKind::UnterminatedLiteral,
"unterminated character literal",
tok.start,
tok.start,
);
}
Some(atom(SyntaxKind::LITERAL, start))
}
TokKind::Integer
| TokKind::BinInt
| TokKind::OctInt
| TokKind::HexInt
| TokKind::Float
| TokKind::Float32
| TokKind::ErrorInvalidNumber
| TokKind::ErrorHexFloatNoP
| TokKind::ErrorIdentifierStart
| TokKind::Unknown
| TokKind::TrueKw
| TokKind::FalseKw => Some(atom(SyntaxKind::LITERAL, start)),
k if is_lone_error_operator(k) => {
let op = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::LoneOperator,
"operator is not a valid value",
op.start,
op.end,
);
Some(error_operator_atom(start))
}
k if is_value_operator(k) || k == TokKind::Question => {
let separated = ctx.skip_ws(start + 1);
if flags.array_mode
&& separated > start + 1
&& ctx.token(separated).map(|t| t.kind) == Some(TokKind::LParen)
{
return Some(atom(SyntaxKind::OPERATOR_ATOM, start));
}
let operand_start = ctx.skip_trivia(start + 1);
if k != TokKind::Question
&& ctx.token(operand_start).map(|t| t.kind) == Some(TokKind::Eq)
{
return Some(atom(SyntaxKind::OPERATOR_ATOM, start));
}
let operand_flags = ExprFlags {
no_range: false,
array_mode: false,
no_juxtapose: true,
..flags
};
let newline_stop = newline_stops_operand(ctx, start, flags);
let operand = if newline_stop {
None
} else {
parse_expr_in(
ctx.tokens(),
operand_start,
PREFIX_BP,
diagnostics,
operand_flags,
)
};
match operand {
Some(operand) => {
let op = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidPrefixOperator,
"invalid operator in prefix position",
op.start,
op.end,
);
let mut events = vec![Event::Start(SyntaxKind::UNARY_EXPR)];
events.extend(error_wrapped_atom(SyntaxKind::OPERATOR_ATOM, start).events);
push_range(&mut events, start + 1, operand.start);
events.extend(operand.events);
finish(&mut events, SyntaxKind::UNARY_EXPR);
Some(ExprParse {
start,
end: operand.end,
events,
})
}
None if matches!(ctx.tokens()[start].kind, TokKind::Question) => {
let op = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::LoneOperator,
"operator is not a valid value",
op.start,
op.end,
);
Some(error_operator_atom(start))
}
None => Some(atom(SyntaxKind::OPERATOR_ATOM, start)),
}
}
_ => None,
}
}
fn error_wrapped_atom(inner: SyntaxKind, start: usize) -> ExprParse {
let mut events = vec![
Event::Start(SyntaxKind::ERROR),
Event::Start(inner),
Event::Tok(start),
];
finish(&mut events, inner);
finish(&mut events, SyntaxKind::ERROR);
ExprParse {
start,
end: start + 1,
events,
}
}
fn keyword_name_error_atom(start: usize) -> ExprParse {
error_wrapped_atom(SyntaxKind::NAME, start)
}
fn is_name_error_keyword(kind: TokKind) -> bool {
kind.is_keyword()
&& !matches!(
kind,
TokKind::MutableKw | TokKind::WhereKw | TokKind::TrueKw | TokKind::FalseKw
)
}
fn char_token_terminated(text: &str) -> bool {
text.len() >= 2 && text.ends_with('\'')
}
fn error_operator_atom(start: usize) -> ExprParse {
error_wrapped_atom(SyntaxKind::OPERATOR_ATOM, start)
}
fn is_closing_block_keyword(kind: TokKind) -> bool {
matches!(
kind,
TokKind::EndKw
| TokKind::ElseKw
| TokKind::ElseifKw
| TokKind::CatchKw
| TokKind::FinallyKw
)
}
pub(super) fn parse_quote_sym(
ctx: &ParserCtx<'_>,
start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
value_position: bool,
end_marker: bool,
inside_brackets: bool,
generator_for_ends: bool,
) -> Option<ExprParse> {
let next_on_line = ctx.skip_ws_and_comments(start + 1);
if value_position
&& matches!(
ctx.token(next_on_line).map(|t| t.kind),
None | Some(
TokKind::RParen
| TokKind::RBracket
| TokKind::RBrace
| TokKind::Comma
| TokKind::Semicolon
)
)
{
return None;
}
let next = ctx.skip_trivia(start + 1);
if value_position
&& next > start + 1
&& ctx
.token(next)
.is_some_and(|t| is_closing_block_keyword(t.kind))
&& !(ctx.token(next).map(|t| t.kind) == Some(TokKind::EndKw) && end_marker)
{
return None;
}
let mut events = vec![Event::Start(SyntaxKind::QUOTE_SYM), Event::Tok(start)];
push_range(&mut events, start + 1, next);
if next > start + 1 {
let colon = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::QuoteColonWhitespace,
"whitespace after `:`",
colon.end,
colon.end,
);
}
match ctx.token(next).map(|t| t.kind)? {
TokKind::LParen
if {
let op = ctx.skip_trivia(next + 1);
is_paren_quotable_op(ctx.token(op).map(|t| t.kind))
&& ctx.token(ctx.skip_trivia(op + 1)).map(|t| t.kind) == Some(TokKind::RParen)
} =>
{
let op = ctx.skip_trivia(next + 1);
let rparen = ctx.skip_trivia(op + 1);
events.push(Event::Start(SyntaxKind::PAREN_EXPR));
push_range(&mut events, next, op);
events.push(Event::Start(SyntaxKind::OPERATOR_ATOM));
events.push(Event::Tok(op));
finish(&mut events, SyntaxKind::OPERATOR_ATOM);
push_range(&mut events, op + 1, rparen + 1);
finish(&mut events, SyntaxKind::PAREN_EXPR);
finish(&mut events, SyntaxKind::QUOTE_SYM);
Some(ExprParse {
start,
end: rparen + 1,
events,
})
}
TokKind::LParen
if ctx
.token(ctx.skip_trivia(next + 1))
.is_some_and(|t| is_closing_block_keyword(t.kind)) =>
{
events.push(Event::Tok(next)); finish(&mut events, SyntaxKind::QUOTE_SYM);
let lparen = &ctx.tokens()[next];
push_diagnostic(
diagnostics,
DiagnosticKind::EmptyQuoteParen,
"expected expression after `:(`",
lparen.end,
lparen.end,
);
Some(ExprParse {
start,
end: next + 1,
events,
})
}
TokKind::LParen => {
let paren = parse_paren(ctx, next, false, generator_for_ends, diagnostics)?;
let end = paren.end;
events.extend(paren.events);
events.push(Event::Finish);
Some(ExprParse { start, end, events })
}
TokKind::StringPrefix if is_var_identifier_start(ctx, next) => {
let end = push_var_macro_name(ctx, &mut events, next, diagnostics)?;
finish(&mut events, SyntaxKind::QUOTE_SYM);
Some(ExprParse { start, end, events })
}
TokKind::At => {
let mac = parse_macro(
ctx,
next,
diagnostics,
ExprFlags {
inside_brackets,
generator_for_ends,
..ExprFlags::default()
},
);
let end = mac.end;
events.extend(mac.events);
finish(&mut events, SyntaxKind::QUOTE_SYM);
Some(ExprParse { start, end, events })
}
TokKind::Ident => {
events.push(Event::Start(SyntaxKind::NAME));
events.push(Event::Tok(next));
events.push(Event::Finish);
events.push(Event::Finish);
Some(ExprParse {
start,
end: next + 1,
events,
})
}
TokKind::Integer
| TokKind::BinInt
| TokKind::OctInt
| TokKind::HexInt
| TokKind::Float
| TokKind::Float32
| TokKind::ErrorInvalidNumber
| TokKind::ErrorHexFloatNoP
| TokKind::Char => {
events.push(Event::Start(SyntaxKind::LITERAL));
events.push(Event::Tok(next));
finish(&mut events, SyntaxKind::LITERAL);
finish(&mut events, SyntaxKind::QUOTE_SYM);
Some(ExprParse {
start,
end: next + 1,
events,
})
}
TokKind::StringDelimOpen | TokKind::CmdDelimOpen => {
let lit = parse_string_literal(ctx, next, diagnostics);
let end = lit.end;
events.extend(lit.events);
finish(&mut events, SyntaxKind::QUOTE_SYM);
Some(ExprParse { start, end, events })
}
_ if ctx
.token(next)
.is_some_and(|t| is_dotted_broadcast_text(t.text)) =>
{
events.push(Event::Start(SyntaxKind::OPERATOR_ATOM));
events.push(Event::Tok(next));
finish(&mut events, SyntaxKind::OPERATOR_ATOM);
finish(&mut events, SyntaxKind::QUOTE_SYM);
Some(ExprParse {
start,
end: next + 1,
events,
})
}
k if is_op_name(k) || is_assignment_op(k) || is_quotable_operator(k) => {
events.push(Event::Tok(next));
events.push(Event::Finish);
Some(ExprParse {
start,
end: next + 1,
events,
})
}
k if k.is_keyword() => {
events.push(Event::Tok(next));
events.push(Event::Finish);
Some(ExprParse {
start,
end: next + 1,
events,
})
}
_ => None,
}
}
fn parse_string_literal(
ctx: &ParserCtx<'_>,
start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let mut i = start;
let mut var_prefix = false;
let mut has_prefix = false;
if ctx.token(i).map(|t| t.kind) == Some(TokKind::StringPrefix) {
has_prefix = true;
var_prefix = ctx.token(i).map(|t| t.text) == Some("var");
i += 1;
}
let single_quote_open = matches!(
ctx.token(i),
Some(t) if t.kind == TokKind::StringDelimOpen && t.text.len() == 1
);
let node = match ctx.token(i).map(|t| t.kind) {
Some(TokKind::CmdDelimOpen) => SyntaxKind::CMD_LITERAL,
Some(TokKind::StringDelimOpen) if var_prefix && single_quote_open => {
SyntaxKind::NONSTANDARD_IDENTIFIER
}
_ => SyntaxKind::STRING_LITERAL,
};
let close_kind = if node == SyntaxKind::CMD_LITERAL {
TokKind::CmdDelimClose
} else {
TokKind::StringDelimClose
};
let mut events = vec![Event::Start(node)];
for idx in start..=i {
events.push(Event::Tok(idx));
}
i += 1;
loop {
match ctx.token(i).map(|t| t.kind) {
Some(TokKind::StringContent) => {
events.push(Event::Tok(i));
i += 1;
}
Some(TokKind::Dollar) => {
i = parse_interpolation(ctx, &mut events, i, true, diagnostics);
}
Some(k) if k == close_kind => {
events.push(Event::Tok(i));
i += 1;
let suffix = ctx.token(i).map(|t| t.kind);
if node == SyntaxKind::NONSTANDARD_IDENTIFIER {
if matches!(
suffix,
Some(
TokKind::StringSuffix
| TokKind::Integer
| TokKind::Float
| TokKind::Float32
| TokKind::BinInt
| TokKind::OctInt
| TokKind::HexInt
)
) {
events.push(Event::Tok(i));
i += 1;
let lit = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::StringSuffixSpace,
"invalid string-macro suffix",
lit.start,
lit.start,
);
}
break;
}
let is_flag = suffix == Some(TokKind::StringSuffix);
let is_numeric = has_prefix
&& matches!(node, SyntaxKind::STRING_LITERAL | SyntaxKind::CMD_LITERAL)
&& matches!(
suffix,
Some(TokKind::Integer | TokKind::Float | TokKind::Float32)
);
if is_flag || is_numeric {
events.push(Event::Tok(i));
i += 1;
}
break;
}
_ => {
let lit = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::UnterminatedLiteral,
"unterminated literal",
lit.start,
lit.start,
);
break;
}
}
}
events.push(Event::Finish);
ExprParse {
start,
end: i,
events,
}
}
pub(super) fn parse_prefix_interpolation(
ctx: &ParserCtx<'_>,
dollar: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let next = dollar + 1;
if matches!(
ctx.token(next).map(|t| t.kind),
Some(TokKind::LParen | TokKind::Ident)
) {
let mut events = Vec::new();
let end = parse_interpolation(ctx, &mut events, dollar, false, diagnostics);
return ExprParse {
start: dollar,
end,
events,
};
}
let mut events = vec![Event::Start(SyntaxKind::INTERPOLATION), Event::Tok(dollar)];
match parse_prefix(ctx, next, diagnostics, ExprFlags::default()) {
Some(operand) => {
push_range(&mut events, next, operand.start);
let end = operand.end;
events.extend(operand.events);
events.push(Event::Finish);
ExprParse {
start: dollar,
end,
events,
}
}
None => {
events.push(Event::Finish);
ExprParse {
start: dollar,
end: next,
events,
}
}
}
}
fn parse_interpolation(
ctx: &ParserCtx<'_>,
events: &mut Vec<Event>,
dollar: usize,
in_string: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> usize {
events.push(Event::Start(SyntaxKind::INTERPOLATION));
events.push(Event::Tok(dollar)); let next = dollar + 1;
match ctx.token(next).map(|t| t.kind) {
Some(TokKind::LParen) => {
let Some(inner) = parse_paren_in(ctx, next, false, true, !in_string, diagnostics)
else {
events.push(Event::Finish);
return next + 1;
};
if in_string
&& matches!(
inner.events.first(),
Some(Event::Start(
SyntaxKind::PAREN_BLOCK | SyntaxKind::TUPLE_EXPR | SyntaxKind::GENERATOR
))
)
{
let dollar_tok = &ctx.tokens()[dollar];
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidInterpolation,
"interpolation expects a single expression",
dollar_tok.start,
dollar_tok.end,
);
}
let end = inner.end;
events.extend(inner.events);
events.push(Event::Finish);
end
}
Some(TokKind::Ident) => {
events.push(Event::Tok(next));
events.push(Event::Finish);
next + 1
}
_ => {
events.push(Event::Finish);
next
}
}
}
fn atom(kind: SyntaxKind, idx: usize) -> ExprParse {
ExprParse {
start: idx,
end: idx + 1,
events: vec![Event::Start(kind), Event::Tok(idx), Event::Finish],
}
}
pub(crate) fn parse_paren(
ctx: &ParserCtx<'_>,
start: usize,
end_marker: bool,
generator_for_ends: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
parse_paren_in(
ctx,
start,
end_marker,
generator_for_ends,
true,
diagnostics,
)
}
fn parse_paren_in(
ctx: &ParserCtx<'_>,
start: usize,
end_marker: bool,
generator_for_ends: bool,
macro_newlines: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let inner_start = ctx.skip_trivia(start + 1);
match ctx.token(inner_start).map(|t| t.kind) {
Some(TokKind::RParen) => {
let mut events = vec![Event::Start(SyntaxKind::TUPLE_EXPR)];
push_range(&mut events, start, inner_start + 1);
events.push(Event::Finish);
return Some(ExprParse {
start,
end: inner_start + 1,
events,
});
}
Some(TokKind::Semicolon) => {
let (events, end) = parse_arg_list(
ctx,
start,
TokKind::RParen,
paren_list_kind(ctx, start),
end_marker,
generator_for_ends,
diagnostics,
);
return Some(ExprParse { start, end, events });
}
_ => {}
}
if ctx.token(inner_start).map(|t| t.kind) == Some(TokKind::Comma) {
return Some(parse_parenthesized_leading_comma(
ctx,
start,
inner_start,
diagnostics,
));
}
if is_paren_value_op(ctx.token(inner_start).map(|t| t.kind)) {
let close = ctx.skip_trivia(inner_start + 1);
if ctx.token(close).map(|t| t.kind) == Some(TokKind::RParen) {
let mut events = vec![Event::Start(SyntaxKind::PAREN_EXPR)];
push_range(&mut events, start, close + 1);
events.push(Event::Finish);
return Some(ExprParse {
start,
end: close + 1,
events,
});
}
}
let diag_mark = diagnostics.len();
let flags = ExprFlags {
inside_brackets: true,
generator_for_ends,
paren_content_start: macro_newlines.then_some(inner_start),
end_marker,
begin_marker: end_marker,
..ExprFlags::default()
};
let Some(inner) = parse_expr_in(ctx.tokens(), inner_start, 0, diagnostics, flags) else {
if ctx.token(inner_start).is_none() {
let open = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::UnclosedParen,
"unclosed `(`",
open.start,
open.end,
);
}
return Some(error_expr_with_range(start, inner_start));
};
let sep = ctx.skip_trivia(inner.end);
if ctx.token(sep).map(|t| t.kind) == Some(TokKind::ForKw) {
let mut scratch = Vec::new();
let mut scratch_diags = Vec::new();
let clauses_end = parse_generator_clauses(ctx, inner.end, &mut scratch, &mut scratch_diags);
if ctx.token(ctx.skip_trivia(clauses_end)).map(|t| t.kind) != Some(TokKind::RParen) {
return Some(parse_parenthesized_generator_junk(
ctx,
start,
inner,
diagnostics,
));
}
return Some(parse_comprehension(
ctx,
start,
inner,
SyntaxKind::GENERATOR,
TokKind::RParen,
diagnostics,
));
}
if matches!(
ctx.token(sep).map(|t| t.kind),
Some(TokKind::Comma | TokKind::Semicolon)
) {
diagnostics.truncate(diag_mark);
let (events, end) = parse_arg_list(
ctx,
start,
TokKind::RParen,
paren_list_kind(ctx, start),
end_marker,
generator_for_ends,
diagnostics,
);
return Some(ExprParse { start, end, events });
}
let mut events = vec![Event::Start(SyntaxKind::PAREN_EXPR)];
push_range(&mut events, start, inner.start);
events.extend(inner.events);
let close = recover_delimited_suffix(ctx, inner.end, &mut events, diagnostics);
if ctx.token(close).map(|t| t.kind) == Some(TokKind::RParen) {
events.push(Event::Tok(close));
events.push(Event::Finish);
Some(ExprParse {
start,
end: close + 1,
events,
})
} else {
let open = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::UnclosedParen,
"unclosed `(`",
open.start,
open.end,
);
events.push(Event::Finish);
Some(ExprParse {
start,
end: close,
events,
})
}
}
fn recover_delimited_suffix(
ctx: &ParserCtx<'_>,
from: usize,
events: &mut Vec<Event>,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> usize {
let start = ctx.skip_trivia(from);
push_range(events, from, start);
let mut end = start;
while ctx.token(end).is_some_and(|t| {
!matches!(
t.kind,
TokKind::RParen | TokKind::RBracket | TokKind::RBrace
) && !is_closing_block_keyword(t.kind)
}) {
end += 1;
}
if end > start {
events.extend(error_expr_with_range(start, end).events);
let token = &ctx.tokens()[start];
push_diagnostic(
diagnostics,
DiagnosticKind::TrailingJunk,
"expected a closing delimiter or separator",
token.start,
token.end,
);
}
end
}
fn parse_parenthesized_leading_comma(
ctx: &ParserCtx<'_>,
open: usize,
junk_start: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let tokens = ctx.tokens();
let mut close = junk_start;
while ctx.token(close).map(|t| t.kind) != Some(TokKind::RParen) && ctx.token(close).is_some() {
close += 1;
}
let mut events = vec![Event::Start(SyntaxKind::PAREN_EXPR), Event::Tok(open)];
push_range(&mut events, open + 1, junk_start);
events.push(Event::Start(SyntaxKind::ERROR));
push_range(&mut events, junk_start, close);
finish(&mut events, SyntaxKind::ERROR);
push_diagnostic(
diagnostics,
DiagnosticKind::TrailingJunk,
"unexpected leading comma in parentheses",
tokens[junk_start].start,
tokens[junk_start].end,
);
let end = if ctx.token(close).map(|t| t.kind) == Some(TokKind::RParen) {
events.push(Event::Tok(close));
close + 1
} else {
let opener = &tokens[open];
push_diagnostic(
diagnostics,
DiagnosticKind::UnclosedParen,
"unclosed `(`",
opener.start,
opener.end,
);
close
};
finish(&mut events, SyntaxKind::PAREN_EXPR);
ExprParse {
start: open,
end,
events,
}
}
fn parse_parenthesized_generator_junk(
ctx: &ParserCtx<'_>,
open: usize,
body: ExprParse,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let tokens = ctx.tokens();
let mut events = vec![Event::Start(SyntaxKind::PAREN_EXPR), Event::Tok(open)];
push_range(&mut events, open + 1, body.start);
events.push(Event::Start(SyntaxKind::GENERATOR));
let mut pos = body.end;
events.extend(body.events);
pos = parse_generator_clauses(ctx, pos, &mut events, diagnostics);
finish(&mut events, SyntaxKind::GENERATOR);
let close = recover_delimited_suffix(ctx, pos, &mut events, diagnostics);
let end = if ctx.token(close).map(|t| t.kind) == Some(TokKind::RParen) {
events.push(Event::Tok(close));
close + 1
} else {
let opener = &tokens[open];
push_diagnostic(
diagnostics,
DiagnosticKind::UnclosedParen,
"unclosed `(`",
opener.start,
opener.end,
);
close
};
finish(&mut events, SyntaxKind::PAREN_EXPR);
ExprParse {
start: open,
end,
events,
}
}
fn parse_postfix_chain(
ctx: &ParserCtx<'_>,
mut lhs: ExprParse,
array_mode: bool,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
loop {
let next = ctx.skip_ws_and_block_comments(lhs.end);
if array_mode
&& next > lhs.end
&& matches!(
ctx.token(next).map(|t| t.kind),
Some(TokKind::LParen | TokKind::LBracket | TokKind::LBrace)
)
{
break;
}
if ctx.token(next).map(|t| t.kind) == Some(TokKind::LParen) && lhs_is_number(ctx, &lhs) {
break;
}
match ctx.token(next).map(|t| t.kind) {
Some(TokKind::LParen) => {
lhs = parse_postfix(
ctx,
lhs,
next,
TokKind::RParen,
SyntaxKind::CALL_EXPR,
end_marker,
diagnostics,
)
}
Some(TokKind::LBracket) => {
lhs = parse_postfix(
ctx,
lhs,
next,
TokKind::RBracket,
SyntaxKind::INDEX_EXPR,
end_marker,
diagnostics,
)
}
Some(TokKind::LBrace) => {
lhs = parse_postfix(
ctx,
lhs,
next,
TokKind::RBrace,
SyntaxKind::CURLY_EXPR,
end_marker,
diagnostics,
)
}
Some(TokKind::Dot)
if ctx.token(ctx.skip_ws(next + 1)).map(|t| t.kind) == Some(TokKind::LParen) =>
{
let lparen = ctx.skip_ws(next + 1);
let (list_events, end) = parse_arg_list(
ctx,
lparen,
TokKind::RParen,
SyntaxKind::ARG_LIST,
end_marker,
true,
diagnostics,
);
let lhs_is_macrocall = matches!(
lhs.events.first(),
Some(Event::Start(SyntaxKind::MACRO_CALL))
);
let mut events = vec![Event::Start(SyntaxKind::DOT_CALL_EXPR)];
events.extend(lhs.events);
push_range(&mut events, lhs.end, next);
events.push(Event::Tok(next)); push_range(&mut events, next + 1, lparen);
if lparen > next + 1 {
let opener = &ctx.tokens()[lparen];
push_diagnostic(
diagnostics,
DiagnosticKind::OpenerWhitespace,
"whitespace before opener",
opener.start,
opener.start,
);
}
if lhs_is_macrocall {
let opener = &ctx.tokens()[lparen];
push_diagnostic(
diagnostics,
DiagnosticKind::MacroDotBroadcast,
"broadcast call on a macro name",
opener.start,
opener.start,
);
}
events.extend(list_events);
events.push(Event::Finish);
lhs = ExprParse {
start: lhs.start,
end,
events,
};
}
Some(TokKind::Dot)
if ctx
.token(ctx.skip_ws_and_block_comments(next + 1))
.map(|t| t.kind)
== Some(TokKind::Transpose) =>
{
let transpose = ctx.skip_ws_and_block_comments(next + 1);
let dot = &ctx.tokens()[next];
if next > lhs.end {
push_diagnostic(
diagnostics,
DiagnosticKind::DotWhitespace,
"whitespace before `.`",
dot.end,
dot.end,
);
}
let prime = &ctx.tokens()[transpose];
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidPostfixOperator,
"invalid dotted postfix operator",
prime.start,
prime.end,
);
let mut events = vec![Event::Start(SyntaxKind::POSTFIX_EXPR)];
events.extend(lhs.events);
push_range(&mut events, lhs.end, next);
events.push(Event::Tok(next));
push_range(&mut events, next + 1, transpose);
events.push(Event::Tok(transpose));
events.push(Event::Finish);
lhs = ExprParse {
start: lhs.start,
end: transpose + 1,
events,
};
}
Some(TokKind::Transpose) => {
let mut events = vec![Event::Start(SyntaxKind::POSTFIX_EXPR)];
events.extend(lhs.events);
push_range(&mut events, lhs.end, next);
events.push(Event::Tok(next));
events.push(Event::Finish);
lhs = ExprParse {
start: lhs.start,
end: next + 1,
events,
};
}
_ => break,
}
}
let next = ctx.skip_ws_and_block_comments(lhs.end);
if ctx.token(next).map(|t| t.kind) == Some(TokKind::DoKw) {
lhs = parse_do_block(ctx, lhs, next, diagnostics);
}
lhs
}
fn parse_postfix(
ctx: &ParserCtx<'_>,
lhs: ExprParse,
open_idx: usize,
close: TokKind,
node: SyntaxKind,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let first_start = ctx.skip_trivia(open_idx + 1);
if ctx.token(first_start).map(|t| t.kind) != Some(close) {
let gen_end_marker = end_marker || close == TokKind::RBracket;
let flags = ExprFlags {
inside_brackets: true,
generator_for_ends: true,
end_marker: gen_end_marker,
begin_marker: gen_end_marker,
..ExprFlags::default()
};
let diag_mark = diagnostics.len();
if let Some(first) = parse_expr_in(ctx.tokens(), first_start, 0, diagnostics, flags)
&& ctx.token(ctx.skip_trivia(first.end)).map(|t| t.kind) == Some(TokKind::ForKw)
{
if matches!(node, SyntaxKind::CALL_EXPR | SyntaxKind::CURLY_EXPR) {
let mut scratch = Vec::new();
let mut scratch_diags = Vec::new();
let clauses_end =
parse_generator_clauses(ctx, first.end, &mut scratch, &mut scratch_diags);
if ctx.token(ctx.skip_trivia(clauses_end)).map(|t| t.kind)
== Some(TokKind::Semicolon)
{
diagnostics.truncate(diag_mark);
return parse_postfix_arg_list(
ctx,
lhs,
open_idx,
close,
node,
end_marker,
diagnostics,
);
}
}
let generator = parse_comprehension(
ctx,
open_idx,
first,
SyntaxKind::GENERATOR,
close,
diagnostics,
);
let node_kind = if node == SyntaxKind::CALL_EXPR {
SyntaxKind::CALL_EXPR
} else {
SyntaxKind::TYPED_COMPREHENSION
};
let mut events = vec![Event::Start(node_kind)];
events.extend(lhs.events);
push_range(&mut events, lhs.end, open_idx);
events.extend(generator.events);
events.push(Event::Finish);
return ExprParse {
start: lhs.start,
end: generator.end,
events,
};
}
diagnostics.truncate(diag_mark);
}
parse_postfix_arg_list(ctx, lhs, open_idx, close, node, end_marker, diagnostics)
}
fn parse_postfix_arg_list(
ctx: &ParserCtx<'_>,
lhs: ExprParse,
open_idx: usize,
close: TokKind,
node: SyntaxKind,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
if close == TokKind::RBracket
&& let Some(typed) = parse_typed_concat(ctx, &lhs, open_idx, end_marker, diagnostics)
{
return typed;
}
let (list_events, end) = parse_arg_list(
ctx,
open_idx,
close,
SyntaxKind::ARG_LIST,
end_marker,
true,
diagnostics,
);
let mut events = vec![Event::Start(node)];
events.extend(lhs.events);
push_range(&mut events, lhs.end, open_idx);
if open_idx > lhs.end {
let opener = &ctx.tokens()[open_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::OpenerWhitespace,
"whitespace before opener",
opener.start,
opener.start,
);
}
events.extend(list_events);
events.push(Event::Finish);
ExprParse {
start: lhs.start,
end,
events,
}
}
fn parse_typed_concat(
ctx: &ParserCtx<'_>,
lhs: &ExprParse,
open_idx: usize,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let diag_mark = diagnostics.len();
let first_start = ctx.skip_trivia(open_idx + 1);
if ctx.token(first_start).map(|t| t.kind) == Some(TokKind::RBracket) {
return None;
}
let wrap = |body: ExprParse| {
let mut events = vec![Event::Start(SyntaxKind::TYPED_MATRIX_EXPR)];
events.extend(lhs.events.iter().cloned());
push_range(&mut events, lhs.end, open_idx);
let end = body.end;
events.extend(body.events);
events.push(Event::Finish);
ExprParse {
start: lhs.start,
end,
events,
}
};
if let Some(empty) = parse_empty_ncat(ctx, open_idx, first_start, BRACKET_CAT) {
return Some(wrap(empty));
}
let first = parse_element(ctx.tokens(), first_start, true, diagnostics)?;
let look = ctx.skip_ws_and_comments(first.end);
match ctx.token(look).map(|t| t.kind) {
None | Some(TokKind::RBracket | TokKind::Comma | TokKind::ForKw) => {
diagnostics.truncate(diag_mark);
None
}
_ => {
let mut body = parse_matrix(ctx, open_idx, first, BRACKET_CAT, end_marker, diagnostics);
let recovered = diagnostics[diag_mark..]
.iter()
.any(|d| d.kind == DiagnosticKind::MatrixKeywordRecovery);
if let (true, Some(Event::Start(k @ SyntaxKind::VECT_EXPR))) =
(recovered, body.events.first_mut())
{
*k = SyntaxKind::MATRIX_EXPR;
}
if matches!(
body.events.first(),
Some(Event::Start(SyntaxKind::VECT_EXPR))
) {
diagnostics.truncate(diag_mark);
None
} else {
Some(wrap(body))
}
}
}
}
fn parse_arg_list(
ctx: &ParserCtx<'_>,
open_idx: usize,
close: TokKind,
list_kind: SyntaxKind,
inherited_end_marker: bool,
generator_for_ends: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> (Vec<Event>, usize) {
let tokens = ctx.tokens();
let end_marker =
inherited_end_marker || (close == TokKind::RBracket && list_kind == SyntaxKind::ARG_LIST);
let begin_marker = end_marker;
let first = ctx.skip_trivia(open_idx + 1);
let flags = ExprFlags {
inside_brackets: true,
generator_for_ends,
end_marker,
begin_marker,
paren_content_start: (matches!(
list_kind,
SyntaxKind::TUPLE_EXPR | SyntaxKind::PAREN_BLOCK
) && ctx
.token(first)
.is_some_and(|t| t.kind != TokKind::Semicolon))
.then_some(first),
..ExprFlags::default()
};
let mut events = vec![Event::Start(list_kind), Event::Tok(open_idx)];
let mut i = open_idx + 1;
let mut in_params = false;
let mut slot_empty = true;
let mut parsed_element = false;
let mut had_commas = false;
let mut had_splat = false;
let mut num_semis = 0;
loop {
while matches!(tokens.get(i).map(|t| t.kind), Some(k) if k.is_trivia()) {
events.push(Event::Tok(i));
i += 1;
}
match tokens.get(i).map(|t| t.kind) {
None => {
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
in_params = false;
}
let opener = &tokens[open_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::UnterminatedArgList,
"unterminated argument list",
opener.start,
opener.start,
);
break;
}
Some(k) if k == close => {
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
in_params = false;
}
events.push(Event::Tok(i));
i += 1;
break;
}
Some(TokKind::Comma) => {
had_commas = true;
if slot_empty && (parsed_element || close == TokKind::RParen) {
let mut j = i;
while let Some(k) = tokens.get(j).map(|t| t.kind) {
if k == close {
break;
}
j += 1;
}
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
in_params = false;
}
events.push(Event::Start(SyntaxKind::ERROR));
push_range(&mut events, i, j);
events.push(Event::Finish);
push_diagnostic(
diagnostics,
DiagnosticKind::TrailingJunk,
"extra comma in list",
tokens[i].start,
tokens[i].end,
);
i = j;
if tokens.get(i).map(|t| t.kind) == Some(close) {
events.push(Event::Tok(i));
i += 1;
}
break;
}
if slot_empty {
push_diagnostic(
diagnostics,
DiagnosticKind::EmptyListSlot,
"missing element before comma",
tokens[i].start,
tokens[i].start,
);
parsed_element = true;
}
events.push(Event::Tok(i));
i += 1;
slot_empty = true;
}
Some(TokKind::Semicolon) => {
num_semis += 1;
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
}
events.push(Event::Start(SyntaxKind::PARAMETERS));
in_params = true;
events.push(Event::Tok(i));
i += 1;
slot_empty = true;
}
Some(TokKind::EndKw) if !end_marker && (parsed_element || close == TokKind::RParen) => {
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
in_params = false;
}
let opener = &tokens[open_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::UnterminatedArgList,
"unterminated argument list",
opener.start,
opener.start,
);
break;
}
Some(_) if !slot_empty => {
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
in_params = false;
}
i = recover_delimited_suffix(ctx, i, &mut events, diagnostics);
if ctx.token(i).map(|t| t.kind) == Some(close) {
events.push(Event::Tok(i));
i += 1;
} else {
let opener = &tokens[open_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::UnterminatedArgList,
"unterminated argument list",
opener.start,
opener.start,
);
}
break;
}
Some(_) => {
let mark = events.len();
i = parse_one_arg(ctx, &mut events, i, flags, diagnostics);
if !parsed_element {
had_splat = matches!(
events.get(mark + 1),
Some(Event::Start(SyntaxKind::SPLAT_EXPR))
);
}
slot_empty = false;
parsed_element = true;
}
}
}
if in_params {
finish(&mut events, SyntaxKind::PARAMETERS);
}
finish(&mut events, list_kind);
if matches!(list_kind, SyntaxKind::TUPLE_EXPR | SyntaxKind::PAREN_BLOCK) {
let initial_semi = ctx
.token(first)
.is_some_and(|t| t.kind == TokKind::Semicolon);
let is_tuple = had_commas
|| (had_splat && num_semis > 0)
|| (initial_semi && (num_semis == 1 || parsed_element));
events[0] = Event::Start(if is_tuple {
SyntaxKind::TUPLE_EXPR
} else {
SyntaxKind::PAREN_BLOCK
});
}
(events, i)
}
fn parse_one_arg(
ctx: &ParserCtx<'_>,
events: &mut Vec<Event>,
i: usize,
flags: ExprFlags,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> usize {
let tokens = ctx.tokens();
let parse_arg_expr = |tokens: &ParserCtx<'_>, start, diagnostics: &mut Vec<ParseDiagnostic>| {
parse_expr_in(tokens, start, 0, diagnostics, flags)
};
if let Some(eq_idx) = kwarg_eq(ctx, i) {
let mut kw = vec![
Event::Start(SyntaxKind::KEYWORD_ARG),
Event::Start(SyntaxKind::NAME),
Event::Tok(i),
Event::Finish,
];
push_range(&mut kw, i + 1, eq_idx);
kw.push(Event::Tok(eq_idx));
let val_start = ctx.skip_trivia(eq_idx + 1);
push_range(&mut kw, eq_idx + 1, val_start);
let end = match parse_arg_expr(tokens, val_start, diagnostics) {
Some(val) => {
kw.extend(val.events);
val.end
}
None => val_start,
};
finish(&mut kw, SyntaxKind::KEYWORD_ARG);
if ctx.token(ctx.skip_trivia(end)).map(|t| t.kind) == Some(TokKind::ForKw) {
events.push(Event::Start(SyntaxKind::GENERATOR));
events.extend(kw);
let gen_end = parse_generator_clauses(ctx, end, events, diagnostics);
finish(events, SyntaxKind::GENERATOR);
gen_end
} else {
events.extend(kw);
end
}
} else if let Some(arg) = parse_arg_expr(tokens, i, diagnostics) {
if ctx.token(ctx.skip_trivia(arg.end)).map(|t| t.kind) == Some(TokKind::ForKw) {
events.push(Event::Start(SyntaxKind::GENERATOR));
events.extend(arg.events);
let end = parse_generator_clauses(ctx, arg.end, events, diagnostics);
finish(events, SyntaxKind::GENERATOR);
end
} else {
events.push(Event::Start(SyntaxKind::ARG));
events.extend(arg.events);
events.push(Event::Finish);
arg.end
}
} else {
events.push(Event::Tok(i));
i + 1
}
}
fn kwarg_eq(ctx: &ParserCtx<'_>, i: usize) -> Option<usize> {
if ctx.token(i).map(|t| t.kind) != Some(TokKind::Ident) || is_reserved_typegroup(ctx, i) {
return None;
}
let eq = ctx.skip_ws(i + 1);
(ctx.token(eq).map(|t| t.kind) == Some(TokKind::Eq)).then_some(eq)
}
fn is_dotted_broadcast_text(text: &str) -> bool {
text.as_bytes().first() == Some(&b'.') && text.len() > 1 && text.as_bytes()[1] != b'.'
}
fn unary_op_paren_is_call(ctx: &ParserCtx<'_>, lparen_idx: usize) -> bool {
let first = ctx.skip_trivia(lparen_idx + 1);
match ctx.token(first).map(|t| t.kind) {
Some(TokKind::RParen) => return true,
Some(TokKind::Semicolon) => return !paren_is_block(ctx, lparen_idx),
_ => {}
}
let mut depth = 0i32;
let mut i = first;
while let Some(tok) = ctx.token(i) {
match tok.kind {
TokKind::LParen | TokKind::LBracket | TokKind::LBrace => depth += 1,
TokKind::RParen | TokKind::RBracket | TokKind::RBrace => {
if depth == 0 {
return false;
}
depth -= 1;
}
TokKind::Comma | TokKind::DotDotDot if depth == 0 => return true,
_ => {}
}
i += 1;
}
false
}
fn paren_list_kind(ctx: &ParserCtx<'_>, lparen_idx: usize) -> SyntaxKind {
if paren_is_block(ctx, lparen_idx) {
SyntaxKind::PAREN_BLOCK
} else {
SyntaxKind::TUPLE_EXPR
}
}
fn paren_is_block(ctx: &ParserCtx<'_>, lparen_idx: usize) -> bool {
let first = ctx.skip_trivia(lparen_idx + 1);
let initial_semi = ctx.token(first).map(|t| t.kind) == Some(TokKind::Semicolon);
let mut depth = 0i32;
let mut had_commas = false;
let mut had_splat = false;
let mut num_semis = 0u32;
let mut num_subexprs = 0u32;
let mut in_subexpr = false;
let mut i = first;
while let Some(tok) = ctx.token(i) {
match tok.kind {
TokKind::LParen | TokKind::LBracket | TokKind::LBrace => {
if !in_subexpr {
num_subexprs += 1;
in_subexpr = true;
}
depth += 1;
}
TokKind::RParen | TokKind::RBracket | TokKind::RBrace => {
if depth == 0 {
break;
}
depth -= 1;
}
TokKind::Comma if depth == 0 => {
had_commas = true;
in_subexpr = false;
}
TokKind::Semicolon if depth == 0 => {
num_semis += 1;
in_subexpr = false;
}
TokKind::DotDotDot if depth == 0 => had_splat = true,
k if !k.is_trivia() && !in_subexpr => {
num_subexprs += 1;
in_subexpr = true;
}
_ => {}
}
i += 1;
}
let is_tuple = had_commas
|| (had_splat && num_semis >= 1)
|| (initial_semi && (num_semis == 1 || num_subexprs > 0));
!is_tuple && num_semis > 0
}
fn is_operator_call_name(kind: TokKind) -> bool {
use TokKind::*;
matches!(
kind,
Star | Slash
| Backslash
| SlashSlash
| Caret
| Percent
| StarPercent
| PlusPlus
| DotDot
| EqEq
| NotEq
| EqEqEq
| NotEqEq
| Lt
| Le
| Gt
| Ge
| Pipe
| Shl
| Shr
| UShr
| PipeGt
| PipeLt
| FatArrow
| LongArrow
| LeftRightArrow
| LeftLongArrow
| DotStar
| DotStarPercent
| DotSlash
| DotBackslash
| DotSlashSlash
| DotCaret
| DotPercent
| DotEqEq
| DotNotEq
| DotEqEqEq
| DotNotEqEq
| DotLt
| DotLe
| DotGt
| DotGe
| DotShl
| DotShr
| DotUShr
| DotSubtype
| DotSupertype
| DotFatArrow
| DotLongArrow
| DotLeftLongArrow
| DotLeftRightArrow
| DotPipeGt
| DotAmp
| DotPipe
| UniArrow
| UniComparison
| UniColon
| UniPlus
| UniTimes
| UniPower
)
}
fn operator_call_lparen(ctx: &ParserCtx<'_>, start: usize, flags: ExprFlags) -> Option<usize> {
let paren_idx = if flags.inside_brackets && !flags.array_mode {
ctx.skip_trivia(start + 1)
} else {
ctx.skip_ws(start + 1)
};
(ctx.token(paren_idx).map(|t| t.kind) == Some(TokKind::LParen)
&& (paren_idx == start + 1 || !flags.array_mode))
.then_some(paren_idx)
}
fn is_curly_operator_name(kind: TokKind) -> bool {
use TokKind::*;
is_operator_call_name(kind)
|| matches!(
kind,
Plus | Minus
| PlusPercent
| MinusPercent
| DotPlus
| DotMinus
| DotPlusPercent
| DotMinusPercent
| Bang
| Tilde
| DotTilde
| Subtype
| Supertype
)
}
fn is_paren_value_op(kind: Option<TokKind>) -> bool {
let Some(k) = kind else { return false };
use TokKind::*;
(is_op_name(k) && !matches!(k, AndAnd | OrOr | Arrow)) || k == Colon
}
fn is_paren_quotable_op(kind: Option<TokKind>) -> bool {
let Some(k) = kind else { return false };
use TokKind::*;
is_op_name(k)
|| is_assignment_op(k)
|| matches!(k, ColonColon | Colon | ColonEq | UniAssign)
|| matches!(k, Dot | DotDotDot | DotAndAnd | DotOrOr)
}
fn parse_ternary(
ctx: &ParserCtx<'_>,
cond: ExprParse,
q_idx: usize,
diagnostics: &mut Vec<ParseDiagnostic>,
flags: ExprFlags,
) -> Result<ExprParse, ExprParse> {
let tokens = ctx.tokens();
let inside_brackets = flags.inside_brackets;
let ws_after = |idx: usize| ctx.token(idx + 1).is_some_and(|t| t.kind.is_trivia());
let q_errors = usize::from(q_idx == cond.end) + usize::from(!ws_after(q_idx));
let then_start = ctx.skip_trivia(q_idx + 1);
let then_flags = ExprFlags {
no_range: true,
array_mode: false,
stmt_comma: false,
..flags
};
let Some(then_br) = parse_expr_in(
tokens,
then_start,
TERNARY_BRANCH_BP,
diagnostics,
then_flags,
) else {
let op = &tokens[q_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingTernaryTrue,
"expected expression after `?`",
op.start,
op.end,
);
return Err(error_expr_to_line_end(tokens, cond.start, q_idx + 1));
};
let colon = if inside_brackets {
ctx.skip_trivia(then_br.end)
} else {
ctx.skip_ws_and_block_comments(then_br.end)
};
let has_colon = ctx.token(colon).map(|t| t.kind) == Some(TokKind::Colon);
let (colon_errors, else_start) = if has_colon {
let errors = usize::from(colon == then_br.end) + usize::from(!ws_after(colon));
(errors, ctx.skip_trivia(colon + 1))
} else {
(1, ctx.skip_trivia(then_br.end))
};
let else_flags = ExprFlags {
array_mode: false,
stmt_comma: false,
..flags
};
let Some(else_br) = parse_expr_in(
tokens,
else_start,
TERNARY_BRANCH_BP,
diagnostics,
else_flags,
) else {
let terminator_is_closer = ctx
.token(else_start)
.is_some_and(|t| is_closing_block_keyword(t.kind));
if terminator_is_closer {
let markers = if has_colon { 1 } else { 2 };
let q_end = tokens[q_idx].end;
for _ in 0..markers {
push_diagnostic(
diagnostics,
DiagnosticKind::IncompleteTernaryIf,
"incomplete ternary recovered as `if`",
q_end,
q_end,
);
}
let mut events = vec![Event::Start(SyntaxKind::TERNARY_EXPR)];
events.extend(cond.events);
push_range(&mut events, cond.end, q_idx);
events.push(Event::Tok(q_idx)); push_range(&mut events, q_idx + 1, then_br.start);
events.extend(then_br.events);
let end = if has_colon {
push_range(&mut events, then_br.end, colon);
events.push(Event::Tok(colon)); colon + 1
} else {
then_br.end
};
events.push(Event::Finish);
return Ok(ExprParse {
start: cond.start,
end,
events,
});
}
if has_colon {
let op = &tokens[colon];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingTernaryFalse,
"expected expression after `:`",
op.start,
op.end,
);
return Err(error_expr_to_line_end(tokens, cond.start, colon + 1));
}
let op = &tokens[q_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::MissingTernaryColon,
"expected `:` in ternary expression",
op.start,
op.end,
);
let mut events = vec![Event::Start(SyntaxKind::TERNARY_EXPR)];
events.extend(cond.events);
push_range(&mut events, cond.end, q_idx);
events.push(Event::Tok(q_idx));
push_range(&mut events, q_idx + 1, then_br.start);
events.extend(then_br.events);
events.push(Event::Finish);
return Ok(ExprParse {
start: cond.start,
end: then_br.end,
events,
});
};
let q_end = tokens[q_idx].end;
for _ in 0..q_errors {
push_diagnostic(
diagnostics,
DiagnosticKind::TernaryQWhitespace,
"whitespace around `?`",
q_end,
q_end,
);
}
let then_end = tokens[then_br.end - 1].end;
for _ in 0..colon_errors {
push_diagnostic(
diagnostics,
DiagnosticKind::TernaryColonWhitespace,
"whitespace around `:`",
then_end,
then_end,
);
}
let mut events = vec![Event::Start(SyntaxKind::TERNARY_EXPR)];
events.extend(cond.events);
push_range(&mut events, cond.end, q_idx);
events.push(Event::Tok(q_idx)); push_range(&mut events, q_idx + 1, then_br.start);
events.extend(then_br.events);
if has_colon {
push_range(&mut events, then_br.end, colon);
events.push(Event::Tok(colon)); push_range(&mut events, colon + 1, else_br.start);
} else {
push_range(&mut events, then_br.end, else_br.start);
}
events.extend(else_br.events);
events.push(Event::Finish);
Ok(ExprParse {
start: cond.start,
end: else_br.end,
events,
})
}