use rowan::GreenNodeBuilder;
use crate::diagnostics::{Diagnostic, DiagnosticCode};
use crate::lexer::{self, LexError, Token};
use crate::syntax::{SyntaxKind, SyntaxNode, TextRange};
pub const DEFAULT_MAX_DEPTH: usize = 128;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct ParseOptions {
pub max_depth: usize,
}
impl Default for ParseOptions {
#[inline]
fn default() -> Self {
Self {
max_depth: DEFAULT_MAX_DEPTH,
}
}
}
impl ParseOptions {
#[inline]
#[must_use]
pub fn with_max_depth(mut self, max_depth: usize) -> Self {
self.max_depth = max_depth;
self
}
}
#[derive(Clone)]
pub struct CstDocument {
green: rowan::GreenNode,
diagnostics: Vec<Diagnostic>,
source_len: usize,
}
impl CstDocument {
#[must_use]
pub fn root(&self) -> SyntaxNode {
SyntaxNode::new_root(self.green.clone())
}
#[must_use]
pub fn diagnostics(&self) -> &[Diagnostic] {
&self.diagnostics
}
#[must_use]
pub fn source_len(&self) -> usize {
self.source_len
}
#[inline]
pub(crate) fn from_green_for_edit(green: rowan::GreenNode) -> Self {
let source_len = usize::from(green.text_len());
Self {
green,
diagnostics: Vec::new(),
source_len,
}
}
}
impl std::fmt::Debug for CstDocument {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CstDocument")
.field("root", &self.root())
.field("diagnostics", &self.diagnostics)
.field("source_len", &self.source_len)
.finish()
}
}
#[must_use]
pub fn parse(src: &str) -> CstDocument {
parse_with_options(src, ParseOptions::default())
}
#[must_use]
pub fn parse_with_options(src: &str, options: ParseOptions) -> CstDocument {
let tokens = lexer::tokenize(src);
Parser::new(src.len(), tokens, options).parse_document()
}
pub fn parse_bytes(bytes: &[u8]) -> Result<CstDocument, LexError> {
let src = lexer::validate_utf8(bytes)?;
Ok(parse(src))
}
struct Spanned<'a> {
tok: Token<'a>,
start: usize,
}
struct Parser<'a> {
tokens: Vec<Spanned<'a>>,
cursor: usize,
builder: GreenNodeBuilder<'static>,
source_len: usize,
diagnostics: Vec<Diagnostic>,
options: ParseOptions,
}
impl<'a> Parser<'a> {
fn new(source_len: usize, tokens: Vec<Token<'a>>, options: ParseOptions) -> Self {
let mut offset = 0usize;
let spanned = tokens
.into_iter()
.map(|tok| {
let start = offset;
offset += tok.text.len();
Spanned { tok, start }
})
.collect();
Self {
tokens: spanned,
cursor: 0,
builder: GreenNodeBuilder::new(),
source_len,
diagnostics: Vec::new(),
options,
}
}
fn parse_document(mut self) -> CstDocument {
self.builder.start_node(rowan_kind(SyntaxKind::Root));
self.eat_trivia();
while self.at(SyntaxKind::Hash) {
self.parse_extension_attr();
self.eat_trivia();
}
if !self.at_eof() {
self.parse_value(0);
}
self.eat_trivia();
if !self.at_eof() {
let start = self.current_offset();
let end = self.source_len;
self.push_diagnostic(
DiagnosticCode::UnexpectedToken,
TextRange::new(start, end),
"unexpected trailing tokens after the top-level value",
);
while !self.at_eof() {
self.bump_into_error();
self.eat_trivia();
}
}
self.builder.finish_node(); let green = self.builder.finish();
CstDocument {
green,
diagnostics: self.diagnostics,
source_len: self.source_len,
}
}
fn parse_value(&mut self, depth: usize) {
self.eat_trivia();
let Some(kind) = self.peek_kind() else {
return;
};
match kind {
SyntaxKind::LParen | SyntaxKind::LBracket | SyntaxKind::LBrace
if depth >= self.options.max_depth =>
{
self.recover_depth_limit();
}
SyntaxKind::LParen => self.parse_tuple_or_struct(None, depth),
SyntaxKind::LBracket => self.parse_list(depth),
SyntaxKind::LBrace => self.parse_map(depth),
SyntaxKind::Ident => self.parse_ident_led(depth),
SyntaxKind::TrueKw | SyntaxKind::FalseKw => self.parse_literal(),
SyntaxKind::Integer
| SyntaxKind::Float
| SyntaxKind::String
| SyntaxKind::RawString
| SyntaxKind::Char => self.parse_literal(),
_ => {
let range = self.current_token_range();
self.push_diagnostic(DiagnosticCode::UnexpectedToken, range, "expected a value");
self.bump_into_error();
}
}
}
fn parse_ident_led(&mut self, depth: usize) {
let next_sig = self.peek_kind_after_first_significant();
match next_sig {
Some(SyntaxKind::LParen) => {
let name_checkpoint = self.builder.checkpoint();
self.bump(); self.parse_tuple_or_struct(Some(name_checkpoint), depth);
}
Some(SyntaxKind::LBrace) => {
self.builder.start_node(rowan_kind(SyntaxKind::EnumVariant));
self.bump(); self.eat_trivia();
self.parse_map_like_braces(depth);
self.builder.finish_node();
}
_ => {
self.builder.start_node(rowan_kind(SyntaxKind::EnumVariant));
self.bump(); self.builder.finish_node();
}
}
}
fn parse_tuple_or_struct(&mut self, name_checkpoint: Option<rowan::Checkpoint>, depth: usize) {
let is_struct = self.parens_contain_struct_fields();
let kind = if is_struct {
SyntaxKind::Struct
} else {
if self.parens_are_empty() {
SyntaxKind::Unit
} else {
SyntaxKind::Tuple
}
};
match name_checkpoint {
Some(cp) => self.builder.start_node_at(cp, rowan_kind(kind)),
None => self.builder.start_node(rowan_kind(kind)),
}
self.eat_trivia();
let open = self.current_offset();
self.expect_bump(SyntaxKind::LParen);
self.eat_trivia();
while !self.at_eof() && !self.at(SyntaxKind::RParen) {
let before = self.cursor;
if is_struct {
self.parse_struct_field(depth);
} else {
self.parse_value(depth + 1);
}
self.eat_trivia();
if self.at(SyntaxKind::Comma) {
self.bump();
self.eat_trivia();
} else if self.at(SyntaxKind::RParen) || self.at_eof() {
break;
} else {
self.recover_unexpected_in_group();
}
if self.cursor == before {
self.recover_unexpected_in_group();
}
}
self.eat_trivia();
self.expect_close(SyntaxKind::RParen, open, "(");
self.builder.finish_node();
}
fn parse_struct_field(&mut self, depth: usize) {
self.builder.start_node(rowan_kind(SyntaxKind::StructField));
self.eat_trivia();
if self.at(SyntaxKind::Ident) {
self.bump();
}
self.eat_trivia();
if self.at(SyntaxKind::Colon) {
self.bump();
}
self.eat_trivia();
if !self.at(SyntaxKind::Comma) && !self.at(SyntaxKind::RParen) && !self.at_eof() {
self.parse_value(depth + 1);
}
self.builder.finish_node();
}
fn parse_list(&mut self, depth: usize) {
self.builder.start_node(rowan_kind(SyntaxKind::List));
self.eat_trivia();
let open = self.current_offset();
self.expect_bump(SyntaxKind::LBracket);
self.eat_trivia();
while !self.at_eof() && !self.at(SyntaxKind::RBracket) {
let before = self.cursor;
self.parse_value(depth + 1);
self.eat_trivia();
if self.at(SyntaxKind::Comma) {
self.bump();
self.eat_trivia();
} else if self.at(SyntaxKind::RBracket) || self.at_eof() {
break;
} else {
self.recover_unexpected_in_group();
}
if self.cursor == before {
self.recover_unexpected_in_group();
}
}
self.eat_trivia();
self.expect_close(SyntaxKind::RBracket, open, "[");
self.builder.finish_node();
}
fn parse_map(&mut self, depth: usize) {
self.builder.start_node(rowan_kind(SyntaxKind::Map));
self.parse_map_like_braces(depth);
self.builder.finish_node();
}
fn parse_map_like_braces(&mut self, depth: usize) {
self.eat_trivia();
let open = self.current_offset();
self.expect_bump(SyntaxKind::LBrace);
self.eat_trivia();
while !self.at_eof() && !self.at(SyntaxKind::RBrace) {
let before = self.cursor;
self.parse_map_entry(depth);
self.eat_trivia();
if self.at(SyntaxKind::Comma) {
self.bump();
self.eat_trivia();
} else if self.at(SyntaxKind::RBrace) || self.at_eof() {
break;
} else {
self.recover_unexpected_in_group();
}
if self.cursor == before {
self.recover_unexpected_in_group();
}
}
self.eat_trivia();
self.expect_close(SyntaxKind::RBrace, open, "{");
}
fn parse_map_entry(&mut self, depth: usize) {
self.builder.start_node(rowan_kind(SyntaxKind::MapEntry));
self.eat_trivia();
if !self.at(SyntaxKind::Colon) && !self.at(SyntaxKind::RBrace) && !self.at_eof() {
self.parse_value(depth + 1);
}
self.eat_trivia();
if self.at(SyntaxKind::Colon) {
self.bump();
}
self.eat_trivia();
if !self.at(SyntaxKind::Comma) && !self.at(SyntaxKind::RBrace) && !self.at_eof() {
self.parse_value(depth + 1);
}
self.builder.finish_node();
}
fn parse_literal(&mut self) {
self.builder.start_node(rowan_kind(SyntaxKind::Literal));
self.bump(); self.builder.finish_node();
}
fn parse_extension_attr(&mut self) {
self.builder
.start_node(rowan_kind(SyntaxKind::ExtensionAttr));
self.expect_bump(SyntaxKind::Hash);
self.eat_trivia();
if self.at(SyntaxKind::Bang) {
self.bump();
}
self.eat_trivia();
if self.at(SyntaxKind::LBracket) {
self.bump();
self.eat_trivia();
let mut depth = 1usize;
while !self.at_eof() && depth > 0 {
match self.peek_kind() {
Some(SyntaxKind::LBracket) => {
depth += 1;
self.bump();
}
Some(SyntaxKind::RBracket) => {
depth -= 1;
self.bump();
}
Some(_) => self.bump(),
None => break,
}
if depth > 0 {
self.eat_trivia();
}
}
}
self.builder.finish_node();
}
fn recover_unexpected_in_group(&mut self) {
self.eat_trivia();
if self.at_eof() {
return;
}
let range = self.current_token_range();
self.push_diagnostic(
DiagnosticCode::UnexpectedToken,
range,
"unexpected token in delimited group",
);
self.bump_into_error();
}
fn recover_depth_limit(&mut self) {
self.eat_trivia();
let start = self.current_offset();
self.push_diagnostic(
DiagnosticCode::NestingDepthExceeded,
TextRange::new(start, self.source_len),
"nesting depth exceeds the configured limit",
);
while !self.at_eof() {
self.bump_into_error();
self.eat_trivia();
}
}
fn peek_kind(&self) -> Option<SyntaxKind> {
self.tokens.get(self.cursor).map(|t| t.tok.kind)
}
fn at(&self, kind: SyntaxKind) -> bool {
self.peek_significant() == Some(kind)
}
fn peek_significant(&self) -> Option<SyntaxKind> {
self.tokens[self.cursor..]
.iter()
.map(|t| t.tok.kind)
.find(|k| !k.is_trivia())
}
fn peek_kind_after_first_significant(&self) -> Option<SyntaxKind> {
let mut sig_seen = 0;
for t in &self.tokens[self.cursor..] {
if t.tok.kind.is_trivia() {
continue;
}
sig_seen += 1;
if sig_seen == 2 {
return Some(t.tok.kind);
}
}
None
}
fn at_eof(&self) -> bool {
self.peek_significant().is_none()
}
fn current_offset(&self) -> usize {
self.tokens
.get(self.cursor)
.map_or(self.source_len, |t| t.start)
}
fn current_token_range(&self) -> TextRange {
for t in &self.tokens[self.cursor..] {
if !t.tok.kind.is_trivia() {
return TextRange::new(t.start, t.start + t.tok.text.len());
}
}
TextRange::new(self.source_len, self.source_len)
}
fn parens_contain_struct_fields(&self) -> bool {
let mut i = self.cursor;
while i < self.tokens.len() && self.tokens[i].tok.kind != SyntaxKind::LParen {
if !self.tokens[i].tok.kind.is_trivia() {
return false;
}
i += 1;
}
if i >= self.tokens.len() {
return false;
}
i += 1; let mut depth = 1usize;
let mut last_significant: Option<SyntaxKind> = None;
while i < self.tokens.len() && depth > 0 {
let k = self.tokens[i].tok.kind;
match k {
SyntaxKind::LParen | SyntaxKind::LBracket | SyntaxKind::LBrace => depth += 1,
SyntaxKind::RParen | SyntaxKind::RBracket | SyntaxKind::RBrace => depth -= 1,
SyntaxKind::Colon
if depth == 1
&& last_significant == Some(SyntaxKind::Ident) =>
{
return true;
}
_ => {}
}
if !k.is_trivia() && depth >= 1 {
last_significant = Some(k);
}
i += 1;
}
false
}
fn parens_are_empty(&self) -> bool {
let mut i = self.cursor;
while i < self.tokens.len() && self.tokens[i].tok.kind != SyntaxKind::LParen {
i += 1;
}
if i >= self.tokens.len() {
return false;
}
i += 1; while i < self.tokens.len() {
let k = self.tokens[i].tok.kind;
if k.is_trivia() {
i += 1;
continue;
}
return k == SyntaxKind::RParen;
}
false
}
fn eat_trivia(&mut self) {
while let Some(spanned) = self.tokens.get(self.cursor) {
if spanned.tok.kind.is_trivia() {
self.builder
.token(rowan_kind(spanned.tok.kind), spanned.tok.text);
self.cursor += 1;
} else {
break;
}
}
}
fn bump(&mut self) {
self.eat_trivia();
if let Some(spanned) = self.tokens.get(self.cursor) {
self.builder
.token(rowan_kind(spanned.tok.kind), spanned.tok.text);
self.cursor += 1;
}
}
fn bump_into_error(&mut self) {
self.eat_trivia();
if let Some(spanned) = self.tokens.get(self.cursor) {
self.builder.start_node(rowan_kind(SyntaxKind::Error));
self.builder
.token(rowan_kind(spanned.tok.kind), spanned.tok.text);
self.builder.finish_node();
self.cursor += 1;
}
}
fn expect_bump(&mut self, kind: SyntaxKind) {
if self.at(kind) {
self.bump();
}
}
fn expect_close(&mut self, kind: SyntaxKind, open_offset: usize, open: &str) {
if self.at(kind) {
self.bump();
} else {
let end = self.current_offset();
self.push_diagnostic(
DiagnosticCode::UnclosedDelimiter,
TextRange::new(open_offset, end),
format!("unclosed delimiter `{open}`"),
);
}
}
fn push_diagnostic(
&mut self,
code: DiagnosticCode,
range: TextRange,
message: impl Into<String>,
) {
debug_assert!(
range.start() <= range.end() && range.end() <= self.source_len,
"diagnostic range must lie within [0, source_len)"
);
self.diagnostics.push(Diagnostic::new(code, range, message));
}
}
#[inline]
fn rowan_kind(kind: SyntaxKind) -> rowan::SyntaxKind {
<crate::syntax::kind::RonLang as rowan::Language>::kind_to_raw(kind)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::diagnostics::Severity;
fn roundtrip(src: &str) -> String {
let doc = parse(src);
doc.root()
.descendant_tokens()
.map(|t| t.text().to_string())
.collect()
}
#[test]
fn roundtrip_covers_all_constructs() {
let inputs = [
"",
" \n\t",
"// comment only\n",
"/* block */",
"42",
"-3.14",
"true",
"false",
"'c'",
"\"hello\\nworld\"",
"r#\"raw \"q\" str\"#",
"()",
"Unit",
"Some(42)",
"Foo(x: 1, y: 2.0)",
"(1, 2, 3)",
"[1, 2, 3,]",
"{ \"a\": 1, \"b\": 2, }",
"{ 1: \"one\", 'c': true }",
"Point(x: 1.0, y: -2.0)",
"Enum::A", "Variant { field: 1 }",
"#![enable(implicit_some)]\nSome(5)",
"#![enable(unwrap_newtypes)]\n#![enable(implicit_some)]\n[1, 2]",
"\u{FEFF}42",
"1\r\n2\r\n",
" Foo( a : [ 1 , 2 ] , b : { 'x' : 'y' } ) // trailing\n",
];
for src in inputs {
assert_eq!(roundtrip(src), src, "round-trip failed for {src:?}");
}
}
#[test]
fn valid_input_has_no_diagnostics() {
for src in [
"Foo(x: 1, y: 2.0)",
"[1, 2, 3,]",
"{ \"a\": 1 }",
"Some(())",
"#![enable(implicit_some)]\nSome(5)",
] {
assert!(
parse(src).diagnostics().is_empty(),
"unexpected diagnostics for {src:?}: {:?}",
parse(src).diagnostics()
);
}
}
#[test]
fn parse_bytes_rejects_non_utf8() {
let bad = [0xFFu8, 0x00];
assert!(parse_bytes(&bad).is_err());
}
#[test]
fn parse_bytes_accepts_bom() {
let doc = parse_bytes("\u{FEFF}1".as_bytes()).unwrap();
let printed: String = doc
.root()
.descendant_tokens()
.map(|t| t.text().to_string())
.collect();
assert_eq!(printed, "\u{FEFF}1");
}
#[test]
fn source_len_matches() {
let src = "Foo(x: 1)";
let doc = parse(src);
assert_eq!(doc.source_len(), src.len());
}
#[test]
fn struct_vs_tuple_classification() {
let s = parse("Foo(x: 1)");
let has_struct = s.root().descendant_tokens().count() > 0
&& s.root().children().any(|n| n.kind() == SyntaxKind::Struct);
assert!(has_struct, "named struct should produce a Struct node");
let t = parse("(1, 2)");
let has_tuple = t.root().children().any(|n| n.kind() == SyntaxKind::Tuple);
assert!(has_tuple, "positional parens should produce a Tuple node");
let u = parse("()");
let has_unit = u.root().children().any(|n| n.kind() == SyntaxKind::Unit);
assert!(has_unit, "empty parens should produce a Unit node");
}
#[test]
fn diagnostic_ranges_are_within_source() {
for src in [
"[1, 2", "Foo(x: 1", "{ \"a\": 1", "@", "[1 @ 2]", "1 2 3", "Foo(x: 1) extra", ] {
let doc = parse(src);
for d in doc.diagnostics() {
assert!(
d.range().start() <= d.range().end(),
"range ordered for {src:?}"
);
assert!(
d.range().end() <= doc.source_len(),
"range within source for {src:?}: {:?} (len {})",
d.range(),
doc.source_len()
);
}
}
}
#[test]
fn recovery_diagnostic_codes_and_severity() {
let unclosed = parse("[1, 2");
assert!(unclosed
.diagnostics()
.iter()
.any(|d| d.code() == DiagnosticCode::UnclosedDelimiter
&& d.severity() == Severity::Error));
let stray = parse("@");
assert!(stray.diagnostics().iter().any(
|d| d.code() == DiagnosticCode::UnexpectedToken && d.severity() == Severity::Error
));
}
#[test]
fn one_diagnostic_per_recovery_point() {
let doc = parse("[1, 2");
let unclosed: Vec<_> = doc
.diagnostics()
.iter()
.filter(|d| d.code() == DiagnosticCode::UnclosedDelimiter)
.collect();
assert_eq!(
unclosed.len(),
1,
"exactly one unclosed-delimiter diagnostic"
);
let stray = parse("@");
assert_eq!(stray.diagnostics().len(), 1);
assert_eq!(
stray.diagnostics()[0].code(),
DiagnosticCode::UnexpectedToken
);
}
#[test]
fn malformed_input_roundtrips() {
for src in [
"[1, 2",
"Foo(x: 1",
"{ \"a\": 1",
"Some(",
"(((",
"}]) ",
"@#$%",
"Foo(x: 1) trailing garbage",
"[1 2 3]", "{a 1, b 2}", "[1, [2, [3", ] {
assert_eq!(
roundtrip(src),
src,
"malformed round-trip failed for {src:?}"
);
}
}
#[test]
fn malformed_input_has_error_nodes() {
let doc = parse("@ stray");
let has_error = doc
.root()
.descendant_tokens()
.any(|t| t.parent().map(|p| p.kind()) == Some(SyntaxKind::Error));
assert!(
has_error,
"expected an Error node for stray top-level tokens"
);
}
#[test]
fn parsing_is_deterministic() {
for src in ["Foo(x: 1, y: [2, 3", "@ junk ] ) }", "{a: 1, b: 2,"] {
let a = parse(src);
let b = parse(src);
let ta: String = a
.root()
.descendant_tokens()
.map(|t| t.text().to_string())
.collect();
let tb: String = b
.root()
.descendant_tokens()
.map(|t| t.text().to_string())
.collect();
assert_eq!(ta, tb);
assert_eq!(
a.diagnostics(),
b.diagnostics(),
"diagnostics differ for {src:?}"
);
}
}
#[test]
fn depth_guard_trips_at_bound_plus_one() {
let depth = 5usize;
let opts = ParseOptions::default().with_max_depth(depth);
let src = format!("{}{}", "[".repeat(depth + 1), "]".repeat(depth + 1));
let doc = parse_with_options(&src, opts);
let printed: String = doc
.root()
.descendant_tokens()
.map(|t| t.text().to_string())
.collect();
assert_eq!(printed, src, "depth-limited tree must round-trip");
assert!(
doc.diagnostics()
.iter()
.any(|d| d.code() == DiagnosticCode::NestingDepthExceeded),
"expected an over-limit diagnostic"
);
}
#[test]
fn depth_guard_silent_below_bound() {
let opts = ParseOptions::default().with_max_depth(10);
let src = "[[[[1]]]]";
let doc = parse_with_options(src, opts);
assert!(!doc
.diagnostics()
.iter()
.any(|d| d.code() == DiagnosticCode::NestingDepthExceeded));
}
}