use super::*;
pub(super) fn array_element_boundary(
ctx: &ParserCtx<'_>,
operand_end: usize,
op_idx: usize,
) -> bool {
let space_before = op_idx > operand_end;
if !space_before {
return false;
}
let Some(op) = ctx.token(op_idx) else {
return false;
};
if !op_can_lead_array_element(op) {
return false;
}
!matches!(
ctx.token(op_idx + 1).map(|t| t.kind),
Some(TokKind::Whitespace | TokKind::Newline) | None
)
}
fn op_can_lead_array_element(op: &Token) -> bool {
matches!(
op.kind,
TokKind::Plus
| TokKind::Minus
| TokKind::PlusPercent
| TokKind::MinusPercent
| TokKind::DotPlus
| TokKind::DotMinus
| TokKind::Tilde
| TokKind::DotTilde
| TokKind::Amp
| TokKind::Colon
| TokKind::Dollar
) && !op.text.chars().next_back().is_some_and(is_op_suffix_char)
}
pub(super) fn parse_element(
tokens: &ParserCtx<'_>,
start: usize,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> Option<ExprParse> {
let flags = ExprFlags {
array_mode: true,
generator_for_ends: true,
end_marker,
begin_marker: end_marker,
..ExprFlags::default()
};
parse_expr_in(tokens, start, 0, diagnostics, flags)
}
fn push_element_arg(events: &mut Vec<Event>, el: ExprParse) -> usize {
let end = el.end;
events.push(Event::Start(SyntaxKind::ARG));
events.extend(el.events);
events.push(Event::Finish);
end
}
fn newline_run_precedes(ctx: &ParserCtx<'_>, look: usize, kind: TokKind) -> bool {
ctx.token(ctx.skip_trivia(look)).map(|t| t.kind) == Some(kind)
}
#[derive(Clone, Copy)]
pub(super) struct CatShape {
close: TokKind,
list: SyntaxKind,
cat: SyntaxKind,
comprehension: SyntaxKind,
}
pub(super) const BRACKET_CAT: CatShape = CatShape {
close: TokKind::RBracket,
list: SyntaxKind::VECT_EXPR,
cat: SyntaxKind::MATRIX_EXPR,
comprehension: SyntaxKind::COMPREHENSION,
};
pub(super) const BRACE_CAT: CatShape = CatShape {
close: TokKind::RBrace,
list: SyntaxKind::BRACES,
cat: SyntaxKind::BRACESCAT_EXPR,
comprehension: SyntaxKind::BRACES_COMPREHENSION,
};
pub(super) fn parse_delimited_literal(
ctx: &ParserCtx<'_>,
open: usize,
end_marker: bool,
shape: CatShape,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let diag_mark = diagnostics.len();
let list = |diagnostics: &mut Vec<ParseDiagnostic>| {
diagnostics.truncate(diag_mark);
let (events, end) = parse_arg_list(
ctx,
open,
shape.close,
shape.list,
end_marker,
true,
diagnostics,
);
ExprParse {
start: open,
end,
events,
}
};
let comprehension = |first, diagnostics: &mut Vec<ParseDiagnostic>| {
parse_comprehension(
ctx,
open,
first,
shape.comprehension,
shape.close,
diagnostics,
)
};
let first_start = ctx.skip_trivia(open + 1);
if ctx.token(first_start).map(|t| t.kind) == Some(shape.close) {
return list(diagnostics);
}
if let Some(empty) = parse_empty_ncat(ctx, open, first_start, shape) {
return empty;
}
let Some(first) = parse_element(ctx.tokens(), first_start, end_marker, diagnostics) else {
return list(diagnostics);
};
let look = ctx.skip_ws_and_comments(first.end);
match ctx.token(look).map(|t| t.kind) {
Some(TokKind::ForKw) => comprehension(first, diagnostics),
None | Some(TokKind::Comma) => list(diagnostics),
Some(k) if k == shape.close => list(diagnostics),
Some(TokKind::Newline) if newline_run_precedes(ctx, look, TokKind::ForKw) => {
comprehension(first, diagnostics)
}
Some(TokKind::Newline) if newline_run_precedes(ctx, look, TokKind::Comma) => {
list(diagnostics)
}
_ => parse_matrix(ctx, open, first, shape, end_marker, diagnostics),
}
}
#[derive(PartialEq)]
enum ArrayOrder {
Unknown,
RowMajor,
ColumnMajor,
}
struct SepRun {
toks: Vec<usize>,
semis: usize,
has_newline: bool,
newline_after_semis: bool,
continuation: bool,
}
impl SepRun {
fn dim(&self, between: bool) -> usize {
if self.continuation {
0
} else if self.semis > 0 {
self.semis
} else if self.has_newline && between {
1
} else {
0
}
}
}
pub(super) fn parse_empty_ncat(
ctx: &ParserCtx<'_>,
lbrk: usize,
first_start: usize,
shape: CatShape,
) -> Option<ExprParse> {
let (close, node_kind) = (shape.close, shape.cat);
let mut q = first_start;
let mut saw_semi = false;
while let Some(k) = ctx.token(q).map(|t| t.kind) {
match k {
TokKind::Semicolon => {
saw_semi = true;
q += 1;
}
_ if k.is_trivia() => q += 1,
_ => break,
}
}
if !saw_semi || ctx.token(q).map(|t| t.kind) != Some(close) {
return None;
}
let mut events = vec![Event::Start(node_kind), Event::Tok(lbrk)];
push_range(&mut events, lbrk + 1, q + 1);
finish(&mut events, node_kind);
Some(ExprParse {
start: lbrk,
end: q + 1,
events,
})
}
pub(super) fn parse_matrix(
ctx: &ParserCtx<'_>,
lbrk: usize,
first: ExprParse,
shape: CatShape,
end_marker: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let (close, comma_kind, matrix_kind) = (shape.close, shape.list, shape.cat);
let tokens = ctx.tokens();
let lead_start = first.start;
let mut elems = vec![first];
let mut seps: Vec<SepRun> = Vec::new();
let mut pos = elems[0].end;
let end = loop {
let mut run = SepRun {
toks: Vec::new(),
semis: 0,
has_newline: false,
newline_after_semis: false,
continuation: false,
};
let mut q = pos;
while let Some(k) = ctx.token(q).map(|t| t.kind) {
match k {
TokKind::Semicolon => {
run.semis += 1;
run.newline_after_semis = false;
}
TokKind::Newline => {
run.has_newline = true;
if run.semis > 0 {
run.newline_after_semis = true;
}
}
TokKind::Whitespace | TokKind::Comment | TokKind::BlockComment => {}
_ => break,
}
run.toks.push(q);
q += 1;
}
match ctx.token(q).map(|t| t.kind) {
None => {
seps.push(run);
break q;
}
Some(k) if k == close => {
seps.push(run);
break q + 1;
}
Some(TokKind::EndKw) if !end_marker => {
seps.push(run);
let anchor = tokens[elems[elems.len() - 1].end - 1].end;
push_diagnostic(
diagnostics,
DiagnosticKind::MatrixKeywordRecovery,
"misplaced `end` in array",
anchor,
anchor,
);
break q;
}
Some(TokKind::At) if run.toks.is_empty() => {
seps.push(run);
let mut j = q;
while let Some(k) = ctx.token(j).map(|t| t.kind) {
if k == close {
break;
}
j += 1;
}
let mut events = vec![Event::Start(SyntaxKind::ERROR)];
push_range(&mut events, q, j);
events.push(Event::Finish);
push_diagnostic(
diagnostics,
DiagnosticKind::TrailingJunk,
"trailing tokens in array",
tokens[q].start,
tokens[q].end,
);
elems.push(ExprParse {
start: q,
end: j,
events,
});
pos = j;
}
_ => {
seps.push(run);
let el = match parse_element(tokens, q, end_marker, diagnostics) {
Some(el) => el,
None => ExprParse {
start: q,
end: q + 1,
events: vec![Event::Tok(q)],
},
};
pos = el.end;
elems.push(el);
}
}
};
let n = elems.len();
let close_idx = if ctx.token(end.saturating_sub(1)).map(|t| t.kind) == Some(close) {
Some(end - 1)
} else {
None
};
let mut order = ArrayOrder::Unknown;
for k in 0..n.saturating_sub(1) {
let is_space = seps[k].semis == 0 && !seps[k].has_newline;
let is_double_semi = seps[k].semis == 2;
let newline_after_semis = seps[k].newline_after_semis;
let mut continuation = false;
let conflict = match order {
ArrayOrder::Unknown => {
if is_space {
order = ArrayOrder::RowMajor;
} else if is_double_semi {
order = ArrayOrder::ColumnMajor;
}
false
}
ArrayOrder::RowMajor => {
if is_double_semi && newline_after_semis {
continuation = true;
false
} else {
is_double_semi
}
}
ArrayOrder::ColumnMajor => is_space,
};
seps[k].continuation = continuation;
if conflict {
let anchor = tokens[elems[k].end - 1].end;
push_diagnostic(
diagnostics,
DiagnosticKind::ArraySeparatorMismatch,
"cannot mix space and `;;` separators in an array",
anchor,
anchor,
);
}
}
let top_d = (0..n.saturating_sub(1))
.map(|k| seps[k].dim(true))
.chain(std::iter::once(seps[n - 1].dim(false)))
.max()
.unwrap_or(0);
let node_kind = if n == 1 && top_d == 0 {
comma_kind
} else {
matrix_kind
};
let mut events = vec![Event::Start(node_kind), Event::Tok(lbrk)];
push_range(&mut events, lbrk + 1, lead_start);
emit_cat_groups(&mut events, &elems, &seps, 0, n, top_d);
for &t in &seps[n - 1].toks {
events.push(Event::Tok(t));
}
if let Some(close_idx) = close_idx {
events.push(Event::Tok(close_idx));
}
finish(&mut events, node_kind);
ExprParse {
start: lbrk,
end,
events,
}
}
fn emit_cat_groups(
events: &mut Vec<Event>,
elems: &[ExprParse],
seps: &[SepRun],
lo: usize,
hi: usize,
split_d: usize,
) {
let mut g = lo;
for k in lo..hi {
let is_boundary = k + 1 < hi && seps[k].dim(true) == split_d;
if is_boundary {
emit_cat_child(events, elems, seps, g, k + 1);
for &t in &seps[k].toks {
events.push(Event::Tok(t));
}
g = k + 1;
} else if k + 1 == hi {
emit_cat_child(events, elems, seps, g, hi);
}
}
}
fn emit_cat_child(
events: &mut Vec<Event>,
elems: &[ExprParse],
seps: &[SepRun],
lo: usize,
hi: usize,
) {
if hi - lo == 1 {
push_element_arg(events, elems[lo].clone());
return;
}
let inner_d = (lo..hi - 1).map(|k| seps[k].dim(true)).max().unwrap_or(0);
events.push(Event::Start(SyntaxKind::MATRIX_ROW));
emit_cat_groups(events, elems, seps, lo, hi, inner_d);
finish(events, SyntaxKind::MATRIX_ROW);
}
pub(super) fn parse_generator_clauses(
ctx: &ParserCtx<'_>,
mut pos: usize,
events: &mut Vec<Event>,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> usize {
loop {
let for_idx = ctx.skip_trivia(pos);
if ctx.token(for_idx).map(|t| t.kind) != Some(TokKind::ForKw) {
break;
}
if for_idx == pos {
let for_tok = &ctx.tokens()[for_idx];
push_diagnostic(
diagnostics,
DiagnosticKind::GluedFor,
"expected whitespace before `for`",
for_tok.start,
for_tok.start,
);
}
push_range(events, pos, for_idx);
events.push(Event::Start(SyntaxKind::FOR_BINDING));
events.push(Event::Tok(for_idx)); pos = parse_for_specs(ctx, for_idx + 1, events, true, diagnostics);
finish(events, SyntaxKind::FOR_BINDING);
let if_idx = ctx.skip_trivia(pos);
if ctx.token(if_idx).map(|t| t.kind) == Some(TokKind::IfKw) {
push_range(events, pos, if_idx);
events.push(Event::Start(SyntaxKind::COMPREHENSION_IF));
events.push(Event::Tok(if_idx)); pos = if_idx + 1;
let cond_start = ctx.skip_trivia(pos);
push_range(events, pos, cond_start);
if let Some(cond) =
parse_expr_in_brackets(ctx.tokens(), cond_start, 0, false, diagnostics)
{
events.extend(cond.events);
pos = cond.end;
} else {
pos = cond_start;
}
finish(events, SyntaxKind::COMPREHENSION_IF);
}
}
pos
}
pub(super) fn parse_comprehension(
ctx: &ParserCtx<'_>,
open: usize,
elem: ExprParse,
node_kind: SyntaxKind,
close: TokKind,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> ExprParse {
let tokens = ctx.tokens();
let mut events = vec![Event::Start(node_kind), Event::Tok(open)];
push_range(&mut events, open + 1, elem.start);
let mut pos = elem.end;
events.extend(elem.events);
pos = parse_generator_clauses(ctx, pos, &mut events, diagnostics);
let close_idx = ctx.skip_trivia(pos);
push_range(&mut events, pos, close_idx);
let end = if ctx.token(close_idx).map(|t| t.kind) == Some(close) {
events.push(Event::Tok(close_idx));
close_idx + 1
} else {
let tok = &tokens[open];
push_diagnostic(
diagnostics,
DiagnosticKind::UnclosedComprehension,
"unclosed comprehension",
tok.start,
tok.end,
);
close_idx
};
finish(&mut events, node_kind);
ExprParse {
start: open,
end,
events,
}
}
fn is_outer_marker(ctx: &ParserCtx<'_>, idx: usize, flags: ExprFlags) -> Option<usize> {
if !ctx
.token(idx)
.is_some_and(|t| t.kind == TokKind::Ident && t.text == "outer")
{
return None;
}
let next = ctx.skip_trivia(idx + 1);
if ctx
.token(next)
.is_some_and(|t| t.kind == TokKind::Eq || is_for_separator_tok(t))
{
return None;
}
let mut probe = Vec::new();
let probe_flags = ExprFlags {
array_mode: true,
..flags
};
if parse_expr_in(ctx.tokens(), idx, COMMA_ITEM_BP, &mut probe, probe_flags)
.is_none_or(|lone| lone.end != idx + 1)
{
return None;
}
let pattern_start = ctx.skip_trivia(idx + 1);
parse_expr_in(
ctx.tokens(),
pattern_start,
COMMA_ITEM_BP,
&mut probe,
flags,
)?;
Some(pattern_start)
}
pub(crate) fn parse_for_specs(
ctx: &ParserCtx<'_>,
mut pos: usize,
events: &mut Vec<Event>,
bracketed: bool,
diagnostics: &mut Vec<ParseDiagnostic>,
) -> usize {
let tokens = ctx.tokens();
loop {
let var_start = ctx.skip_trivia(pos);
push_range(events, pos, var_start);
let var_flags = ExprFlags {
inside_brackets: bracketed,
for_spec_var: true,
..ExprFlags::default()
};
let outer = is_outer_marker(ctx, var_start, var_flags);
let (var_bp, pattern_start) = match outer {
Some(after_kw) => {
events.push(Event::Start(SyntaxKind::OUTER_BINDING));
events.push(Event::Tok(var_start));
push_range(events, var_start + 1, after_kw);
(COMMA_ITEM_BP, after_kw)
}
None => (0, var_start),
};
if let Some(var) = parse_expr_in(tokens, pattern_start, var_bp, diagnostics, var_flags) {
events.extend(var.events);
pos = var.end;
} else {
pos = pattern_start;
}
if outer.is_some() {
finish(events, SyntaxKind::OUTER_BINDING);
}
let in_idx = ctx.skip_trivia(pos);
if ctx
.token(in_idx)
.is_some_and(|t| is_for_separator_tok(t) || (outer.is_some() && t.kind == TokKind::Eq))
{
push_range(events, pos, in_idx);
events.push(Event::Tok(in_idx));
pos = in_idx + 1;
let iter_start = ctx.skip_trivia(pos);
push_range(events, pos, iter_start);
let iter = if bracketed {
parse_expr_in_brackets(tokens, iter_start, 0, false, diagnostics)
} else {
parse_expr(tokens, iter_start, 0, diagnostics)
};
if let Some(iter) = iter {
events.extend(iter.events);
pos = iter.end;
} else {
pos = iter_start;
}
}
let comma_idx = ctx.skip_trivia(pos);
if ctx.token(comma_idx).map(|t| t.kind) == Some(TokKind::Comma) {
push_range(events, pos, comma_idx);
events.push(Event::Tok(comma_idx));
pos = comma_idx + 1;
continue;
}
break;
}
pos
}