use crate::parser::ParseOptions;
use crate::parser::context::ParserCtx;
use crate::parser::events::Event;
use crate::parser::expr::parse_stmt;
use crate::parser::lexer::{TokKind, Token, lex};
use crate::parser::tree_builder::{build_tree, syntax_kind_for};
use crate::syntax::{SyntaxKind, SyntaxNode};
pub use crate::parser::diagnostics::ParseDiagnostic;
use crate::parser::diagnostics::{DiagnosticKind, push_diagnostic};
#[derive(Debug, Clone)]
pub struct ParseOutput {
pub cst: SyntaxNode,
pub diagnostics: Vec<ParseDiagnostic>,
}
pub fn parse(text: &str) -> ParseOutput {
parse_with_options(text, ParseOptions::default())
}
pub fn parse_with_options(text: &str, options: ParseOptions) -> ParseOutput {
let tokens = lex_for_options(text, options);
let ctx = ParserCtx::with_options(&tokens, options);
let mut diagnostics = Vec::new();
let mut events = Vec::new();
let mut i = 0usize;
while i < tokens.len() {
if tokens[i].kind.is_trivia() {
events.push(Event::Tok(i));
i += 1;
continue;
}
let mut line = Vec::new();
let mut has_semicolon = false;
let mut leftover_mark: Option<usize> = None;
let mut first_is_doc_string = false;
let mut doc_no_target = false;
let line_has_semicolon = rest_of_line_has_semicolon(&tokens, i);
while i < tokens.len() {
match tokens[i].kind {
TokKind::Newline => break,
TokKind::Semicolon => {
has_semicolon = true;
line.push(Event::Tok(i));
i += 1;
}
k if k.is_trivia() => {
line.push(Event::Tok(i));
i += 1;
}
_ => {
if !line_has_semicolon
&& leftover_mark.is_some()
&& (!first_is_doc_string || doc_no_target)
{
line.push(Event::Tok(i));
i += 1;
} else if let Some(expr) = parse_stmt(&ctx, i, &mut diagnostics) {
if leftover_mark.is_none() {
first_is_doc_string = stmt_is_doc_string(&expr.events, &tokens);
}
line.extend(expr.events);
i = expr.end;
leftover_mark.get_or_insert(line.len());
} else if leftover_mark.is_none()
&& is_close_delimiter_tok(tokens[i].kind)
&& !rest_of_line_has_semicolon(&tokens, i)
{
line.push(Event::Start(SyntaxKind::ERROR));
line.push(Event::Finish);
let run_start = tokens[i].start;
line.push(Event::Start(SyntaxKind::ERROR));
while i < tokens.len() && tokens[i].kind != TokKind::Newline {
line.push(Event::Tok(i));
i += 1;
}
line.push(Event::Finish);
push_diagnostic(
&mut diagnostics,
DiagnosticKind::StrayCloser,
"stray closing delimiter",
run_start,
tokens[i - 1].end,
);
} else if leftover_mark.is_none() && is_stray_block_keyword_tok(tokens[i].kind)
{
line.push(Event::Start(SyntaxKind::ERROR));
line.push(Event::Tok(i));
line.push(Event::Finish);
push_diagnostic(
&mut diagnostics,
DiagnosticKind::StrayKeyword,
"unexpected block keyword",
tokens[i].start,
tokens[i].end,
);
i += 1;
leftover_mark = Some(line.len());
} else {
if first_is_doc_string && leftover_mark.is_some() {
doc_no_target = true;
}
line.push(Event::Tok(i));
i += 1;
}
}
}
}
if has_semicolon {
events.push(Event::Start(SyntaxKind::TOPLEVEL_SEMICOLON));
events.extend(line);
events.push(Event::Finish);
} else if let Some(mark) = leftover_mark.filter(|&m| {
let tail = &line[m..];
let defer_to_doc = first_is_doc_string && leftover_starts_with_subtree(tail, &tokens);
!defer_to_doc && tail.iter().any(|e| is_significant_event(e, &tokens))
}) {
events.extend(line[..mark].iter().cloned());
let tail = &line[mark..];
let lead = tail
.iter()
.take_while(|e| !is_significant_event(e, &tokens))
.count();
events.extend(tail[..lead].iter().cloned());
let first_junk = tail[lead..].iter().find_map(|e| match e {
Event::Tok(idx) => Some(*idx),
_ => None,
});
events.push(Event::Start(SyntaxKind::ERROR));
events.extend(tail[lead..].iter().cloned());
events.push(Event::Finish);
if let Some(idx) = first_junk {
push_diagnostic(
&mut diagnostics,
DiagnosticKind::TrailingJunk,
"trailing tokens after statement",
tokens[idx].start,
tokens[idx].end,
);
}
} else {
events.extend(line);
}
}
finish_parse(&tokens, &events, diagnostics, options)
}
pub(super) fn parse_block_fragment(
text: &str,
owner: SyntaxKind,
options: ParseOptions,
) -> Option<ParseOutput> {
let tokens = lex_for_options(text, options);
let ctx = ParserCtx::with_options(&tokens, options);
let mut diagnostics = Vec::new();
let mut events = vec![Event::Start(owner)];
let end = super::structural::run_block_fragment(&ctx, &mut events, owner, &mut diagnostics)?;
if end != tokens.len() {
return None;
}
events.push(Event::Finish);
let mut output = finish_parse(&tokens, &events, diagnostics, options);
output.cst = output.cst.first_child()?.first_child()?;
Some(output)
}
fn finish_parse(
tokens: &[Token],
events: &[Event],
mut diagnostics: Vec<ParseDiagnostic>,
options: ParseOptions,
) -> ParseOutput {
let events = fold_docstrings(events, tokens);
let cst = build_tree(tokens, &events);
flag_invalid_const_decls(&cst, &mut diagnostics);
flag_invalid_function_signatures(&cst, &mut diagnostics);
flag_invalid_catch_vars(&cst, &mut diagnostics);
flag_invalid_export_items(&cst, &mut diagnostics);
flag_numeric_overflow(&cst, &mut diagnostics);
if options.syntax_edition == crate::parser::SyntaxEdition::Julia113 {
for token in cst
.descendants_with_tokens()
.filter_map(|el| el.into_token())
{
if matches!(
token.kind(),
SyntaxKind::PLUS_PERCENT
| SyntaxKind::MINUS_PERCENT
| SyntaxKind::STAR_PERCENT
| SyntaxKind::PLUS_PERCENT_EQ
| SyntaxKind::MINUS_PERCENT_EQ
| SyntaxKind::STAR_PERCENT_EQ
| SyntaxKind::DOT_PLUS_PERCENT
| SyntaxKind::DOT_MINUS_PERCENT
| SyntaxKind::DOT_STAR_PERCENT
| SyntaxKind::DOT_PLUS_PERCENT_EQ
| SyntaxKind::DOT_MINUS_PERCENT_EQ
| SyntaxKind::DOT_STAR_PERCENT_EQ
) && !token
.text()
.chars()
.next_back()
.is_some_and(super::lexer::is_op_suffix_char)
{
let at = usize::from(token.text_range().start());
push_diagnostic(
&mut diagnostics,
DiagnosticKind::UnsupportedEdition,
"wrapping arithmetic requires syntax edition 1.14",
at,
at,
);
}
}
}
ParseOutput { cst, diagnostics }
}
fn flag_numeric_overflow(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
for literal in cst
.descendants()
.filter(|node| node.kind() == SyntaxKind::LITERAL)
{
let Some(number) = literal
.children_with_tokens()
.filter_map(|element| element.into_token())
.find(|token| matches!(token.kind(), SyntaxKind::FLOAT | SyntaxKind::FLOAT32))
else {
continue;
};
if !decimal_float_overflows(number.text(), number.kind()) {
continue;
}
let range = literal.text_range();
push_diagnostic(
diagnostics,
DiagnosticKind::NumericOverflow,
"overflow in floating point literal",
usize::from(range.start()),
usize::from(range.end()),
);
}
}
fn decimal_float_overflows(text: &str, kind: SyntaxKind) -> bool {
if text.contains("0x") || text.contains("0X") {
return false;
}
let mut parseable = text.replace('_', "");
if kind == SyntaxKind::FLOAT32
&& let Some(marker) = parseable.find(['f', 'F'])
{
parseable.replace_range(marker..=marker, "e");
}
match kind {
SyntaxKind::FLOAT32 => parseable
.parse::<f32>()
.is_ok_and(|value| !value.is_finite()),
SyntaxKind::FLOAT => parseable
.parse::<f64>()
.is_ok_and(|value| !value.is_finite()),
_ => false,
}
}
fn flag_invalid_export_items(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
for stmt in cst
.descendants()
.filter(|n| n.kind() == SyntaxKind::EXPORT_STMT)
{
for item in stmt
.children()
.filter(|c| matches!(c.kind(), SyntaxKind::PAREN_EXPR | SyntaxKind::TUPLE_EXPR))
{
if !export_paren_is_symbol(&item) {
let pos = usize::from(item.text_range().start());
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidExportItem,
"invalid `export` item: expected a single symbol",
pos,
pos,
);
}
}
}
}
fn export_paren_is_symbol(item: &SyntaxNode) -> bool {
if item.kind() != SyntaxKind::PAREN_EXPR {
return false;
}
let mut significant = item.children_with_tokens().filter(|el| {
!matches!(
el.kind(),
SyntaxKind::WHITESPACE
| SyntaxKind::NEWLINE
| SyntaxKind::COMMENT
| SyntaxKind::BLOCK_COMMENT
| SyntaxKind::LPAREN
| SyntaxKind::RPAREN
)
});
let Some(first) = significant.next() else {
return false;
};
if significant.next().is_some() {
return false;
}
match first {
rowan::NodeOrToken::Node(n) => match n.kind() {
SyntaxKind::NAME | SyntaxKind::NONSTANDARD_IDENTIFIER | SyntaxKind::INTERPOLATION => {
true
}
SyntaxKind::OPERATOR_ATOM => !export_operator_atom_is_dotted(&n),
_ => false,
},
rowan::NodeOrToken::Token(_) => true,
}
}
fn export_operator_atom_is_dotted(atom: &SyntaxNode) -> bool {
atom.children_with_tokens()
.filter_map(|el| el.into_token())
.find(|t| {
!matches!(
t.kind(),
SyntaxKind::WHITESPACE
| SyntaxKind::NEWLINE
| SyntaxKind::COMMENT
| SyntaxKind::BLOCK_COMMENT
)
})
.is_some_and(|t| {
let bytes = t.text().as_bytes();
bytes.first() == Some(&b'.') && bytes.len() > 1 && bytes[1] != b'.'
})
}
fn flag_invalid_const_decls(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
for node in cst
.descendants()
.filter(|n| n.kind() == SyntaxKind::CONST_STMT)
{
if !const_decl_is_assignment(&node) && !is_struct_const_field(&node) {
let pos = usize::from(node.text_range().start());
push_diagnostic(
diagnostics,
DiagnosticKind::ConstNotAssignment,
"expected assignment after `const`",
pos,
pos,
);
}
}
}
fn flag_invalid_function_signatures(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
for node in cst
.descendants()
.filter(|n| matches!(n.kind(), SyntaxKind::FUNCTION_DEF | SyntaxKind::MACRO_DEF))
{
let Some(sig) = node.children().find(|c| c.kind() == SyntaxKind::SIGNATURE) else {
continue;
};
let sig_is_bare_name = sig
.first_child()
.is_some_and(|inner| is_bare_signature_name(&inner));
let invalid_bare_name = sig_is_bare_name
&& (!function_body_is_empty(&node) || invalid_forward_operator_name(&sig));
let invalid_parenthesized_macro = invalid_parenthesized_macro_signature(&sig);
if invalid_bare_name || invalid_parenthesized_macro {
let pos = usize::from(sig.text_range().start());
push_diagnostic(
diagnostics,
DiagnosticKind::InvalidFunctionSignature,
if invalid_bare_name {
"invalid function signature: expected a call, not a bare name"
} else {
"invalid function signature: macro call or interpolation cannot bind a parameter"
},
pos,
pos,
);
}
}
}
fn invalid_parenthesized_macro_signature(signature: &SyntaxNode) -> bool {
let Some(tuple) = signature
.first_child()
.filter(|node| node.kind() == SyntaxKind::TUPLE_EXPR)
else {
return false;
};
if tuple.children().count() != 1
|| tuple
.children_with_tokens()
.any(|element| element.kind() == SyntaxKind::COMMA)
{
return false;
}
let mut child = tuple.first_child();
while child
.as_ref()
.is_some_and(|node| node.kind() == SyntaxKind::PAREN_EXPR)
{
child = child.and_then(|node| node.first_child());
}
child.is_some_and(|node| {
matches!(
node.kind(),
SyntaxKind::MACRO_CALL | SyntaxKind::INTERPOLATION
)
})
}
fn invalid_forward_operator_name(signature: &SyntaxNode) -> bool {
let Some(inner) = signature.first_child() else {
return false;
};
if inner.kind() == SyntaxKind::ERROR {
return true;
}
if inner.kind() != SyntaxKind::OPERATOR_ATOM {
return false;
}
inner.first_token().is_some_and(|token| {
let text = token.text();
text == "&" || (text.starts_with('.') && !text.starts_with(".."))
})
}
pub(super) fn is_bare_signature_name(node: &SyntaxNode) -> bool {
match node.kind() {
SyntaxKind::NAME
| SyntaxKind::NONSTANDARD_IDENTIFIER
| SyntaxKind::INTERPOLATION
| SyntaxKind::OPERATOR_ATOM => true,
SyntaxKind::ERROR => node
.first_child()
.is_some_and(|inner| inner.kind() == SyntaxKind::OPERATOR_ATOM),
_ => false,
}
}
fn flag_invalid_catch_vars(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
for node in cst
.descendants()
.filter(|n| n.kind() == SyntaxKind::CATCH_CLAUSE)
{
let Some(var) = node.children().find(|c| c.kind() != SyntaxKind::BLOCK) else {
continue;
};
if !matches!(
var.kind(),
SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::NONSTANDARD_IDENTIFIER
) {
let pos = usize::from(var.text_range().start());
push_diagnostic(
diagnostics,
DiagnosticKind::CatchVarNotIdentifier,
"catch variable must be an identifier",
pos,
pos,
);
}
}
}
fn function_body_is_empty(node: &SyntaxNode) -> bool {
match node.children().find(|c| c.kind() == SyntaxKind::BLOCK) {
Some(block) => {
block.first_child().is_none()
&& !block
.children_with_tokens()
.any(|el| el.kind() == SyntaxKind::SEMICOLON)
}
None => true,
}
}
fn is_struct_const_field(node: &SyntaxNode) -> bool {
node.parent()
.filter(|b| b.kind() == SyntaxKind::BLOCK)
.and_then(|b| b.parent())
.is_some_and(|p| p.kind() == SyntaxKind::STRUCT_DEF)
}
fn const_decl_is_assignment(node: &SyntaxNode) -> bool {
let mut body = node.first_child();
while let Some(n) = body {
match n.kind() {
SyntaxKind::GLOBAL_STMT | SyntaxKind::LOCAL_STMT => body = n.first_child(),
SyntaxKind::ASSIGNMENT_EXPR => {
return n
.children_with_tokens()
.filter_map(|el| el.into_token())
.any(|t| t.kind() == SyntaxKind::EQ);
}
_ => return false,
}
}
false
}
fn is_close_delimiter_tok(kind: TokKind) -> bool {
matches!(kind, TokKind::RParen | TokKind::RBracket | TokKind::RBrace)
}
fn is_stray_block_keyword_tok(kind: TokKind) -> bool {
matches!(
kind,
TokKind::EndKw
| TokKind::ElseKw
| TokKind::ElseifKw
| TokKind::CatchKw
| TokKind::FinallyKw
)
}
fn rest_of_line_has_semicolon(tokens: &[Token], start: usize) -> bool {
tokens[start..]
.iter()
.take_while(|t| t.kind != TokKind::Newline)
.any(|t| t.kind == TokKind::Semicolon)
}
fn is_significant_event(event: &Event, tokens: &[Token]) -> bool {
match event {
Event::Start(_) => true,
Event::Tok(idx) => !tokens[*idx].kind.is_trivia(),
Event::Finish => false,
}
}
fn leftover_starts_with_subtree(tail: &[Event], tokens: &[Token]) -> bool {
matches!(
tail.iter().find(|e| is_significant_event(e, tokens)),
Some(Event::Start(_))
)
}
pub(super) fn stmt_is_doc_string(events: &[Event], tokens: &[Token]) -> bool {
matches!(
events.first(),
Some(Event::Start(SyntaxKind::STRING_LITERAL))
) && string_is_doc_eligible(&events[1..], tokens)
}
fn is_doc_container(kind: SyntaxKind) -> bool {
matches!(kind, SyntaxKind::BLOCK | SyntaxKind::TOPLEVEL_SEMICOLON)
}
fn string_is_doc_eligible(inner: &[Event], tokens: &[Token]) -> bool {
match inner.first() {
Some(Event::Tok(idx)) => syntax_kind_for(tokens[*idx].kind) != SyntaxKind::STRING_PREFIX,
_ => true,
}
}
fn fold_docstrings(events: &[Event], tokens: &[Token]) -> Vec<Event> {
let extents = subtree_extents(events);
let mut marks = DocMarks {
open: vec![false; events.len()],
close: vec![false; events.len()],
folds: 0,
};
mark_doc_folds(events, tokens, &extents, 0, events.len(), true, &mut marks);
if marks.folds == 0 {
return events.to_vec();
}
let mut out = Vec::with_capacity(events.len() + 2 * marks.folds);
for (i, event) in events.iter().enumerate() {
if marks.open[i] {
out.push(Event::Start(SyntaxKind::DOC));
}
out.push(event.clone());
if marks.close[i] {
out.push(Event::Finish);
}
}
out
}
struct DocMarks {
open: Vec<bool>,
close: Vec<bool>,
folds: usize,
}
fn subtree_extents(events: &[Event]) -> Vec<usize> {
let mut extents = vec![usize::MAX; events.len()];
let mut open: Vec<usize> = Vec::new();
for (i, event) in events.iter().enumerate() {
match event {
Event::Start(_) => open.push(i),
Event::Finish => {
if let Some(start) = open.pop() {
extents[start] = i;
}
}
Event::Tok(_) => {}
}
}
extents
}
fn mark_doc_folds(
events: &[Event],
tokens: &[Token],
extents: &[usize],
start: usize,
end: usize,
is_container: bool,
marks: &mut DocMarks,
) {
let mut i = start;
while i < end {
let Event::Start(kind) = events[i] else {
i += 1;
continue;
};
let stop = extents[i].min(end);
mark_doc_folds(
events,
tokens,
extents,
i + 1,
stop,
is_doc_container(kind),
marks,
);
if is_container
&& kind == SyntaxKind::STRING_LITERAL
&& string_is_doc_eligible(&events[i + 1..stop], tokens)
&& let Some(target) = doc_target(events, tokens, stop + 1, end)
{
let target_end = extents[target].min(end);
mark_doc_folds(
events,
tokens,
extents,
target + 1,
target_end,
is_doc_container(as_start_kind(&events[target])),
marks,
);
marks.open[i] = true;
marks.close[target_end] = true;
marks.folds += 1;
i = target_end + 1;
continue;
}
i = stop + 1;
}
}
fn as_start_kind(event: &Event) -> SyntaxKind {
match event {
Event::Start(kind) => *kind,
_ => SyntaxKind::ERROR,
}
}
fn doc_target(events: &[Event], tokens: &[Token], from: usize, end: usize) -> Option<usize> {
let mut newlines = 0;
let mut j = from;
while j < end {
match events[j] {
Event::Start(SyntaxKind::ERROR) => return None,
Event::Start(_) => return Some(j),
Event::Tok(t) => {
let kind = tokens[t].kind;
if kind == TokKind::Newline {
newlines += 1;
if newlines > 1 {
return None;
}
} else if !kind.is_trivia() {
return None;
}
j += 1;
}
Event::Finish => return None,
}
}
None
}
pub fn reconstruct(text: &str) -> String {
parse(text)
.cst
.descendants_with_tokens()
.filter_map(|el| el.into_token())
.map(|tok| tok.text().to_string())
.collect()
}
pub(super) fn lex_for_options(text: &str, _options: ParseOptions) -> Vec<Token<'_>> {
lex(text)
}
#[cfg(test)]
mod tests {
use super::*;
fn assert_lossless(input: &str) {
assert_eq!(reconstruct(input), input, "not lossless: {input:?}");
}
#[test]
fn lossless_corpus() {
for input in [
"",
"x = 1\n",
"y = a + b * c\n",
"f(a, b, c)\n",
"v[1]\n",
"a[end]\n",
"a[end - 1]\n",
"a[2:end]\n",
"m[end, end]\n",
"z = (a + b) / 2\n",
"function g(x)\n x ^ 2\nend\n",
"if a >= b\n a\nelseif c\n c\nelse\n b\nend\n",
"begin\n a\n b\nend\n",
"map(xs) do x\n x + 1\nend\n",
"open(\"f\") do\n read()\nend\n",
"# a comment\nx = 1 # trailing\n",
"#= block =#\nq = 2\n",
"obj.field\n",
"x::Int\n",
"Vector{T}\n",
"foo(x::T) where {T<:Number} = x\n",
"f(a, b; c=1, d=2)\n",
"g(args...; kwargs...)\n",
"y = a .+ b .* c\n",
"z = .-x\n",
"a .= f.(b, c)\n",
"t = (a, b)\n",
"u = (a,)\n",
"e = ()\n",
"nt = (x = 1, y = 2)\n",
"y = a ? b : c\n",
"z = a == b ? x + 1 : x - 1\n",
"w = a ? b : c ? d : e\n",
"v = [1, 2, 3]\n",
"e = []\n",
"m = [1 2; 3 4]\n",
"s = [1 +2]\n",
"c = [1; 2; 3]\n",
"q = [x for x in xs]\n",
"r = [x^2 for x in 1:10 if x > 2]\n",
"g = (x for x in xs)\n",
"a; b\n",
"a;;;b;;\n",
";a\n",
"@assert nl > 1 # trailing comment\nnx = 2\n",
"@foo x # c\n",
"x = @bind 🐶 html\"<input>\"\n",
"begin @m 10 end\n",
] {
assert_lossless(input);
}
}
#[test]
fn groups_top_level_semicolon_line() {
let out = parse("a; b\nc\n");
let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
use crate::syntax::SyntaxKind::{NAME, TOPLEVEL_SEMICOLON};
assert_eq!(kinds, vec![TOPLEVEL_SEMICOLON, NAME]);
}
#[test]
fn builds_expected_top_level_shape() {
let out = parse("x = 1 + 2\n");
let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
assert_eq!(kinds, vec![crate::syntax::SyntaxKind::ASSIGNMENT_EXPR]);
assert!(out.diagnostics.is_empty());
}
#[test]
fn overflowing_decimal_floats_are_parse_errors() {
let out = parse("x=1e400\ny=1f400\nz=1e-4000\n");
assert_eq!(
out.diagnostics
.iter()
.map(|diagnostic| (
diagnostic.start,
diagnostic.end,
diagnostic.message.as_str()
))
.collect::<Vec<_>>(),
vec![
(2, 7, "overflow in floating point literal"),
(10, 15, "overflow in floating point literal"),
]
);
}
}