use crate::VERSION;
use crate::frontend::ast::*;
use crate::frontend::parser::{Cst, NodeRef};
use crate::frontend::sema::*;
use rustc_hash::{FxHashMap, FxHashSet};
use std::collections::{BTreeMap, BTreeSet};
use std::io::{BufWriter, Write};
use std::path::Path;
pub fn snake_to_pascal_case(name: &str) -> String {
let mut res = String::new();
let mut upper = true;
for c in name.chars() {
if upper {
res.push(c.to_ascii_uppercase());
upper = false;
} else if c == '_' {
upper = true;
} else {
res.push(c);
}
}
res
}
fn escape(sym: &str) -> String {
sym.replace(r"\\", r"\")
.replace(r"\'", "'")
.escape_default()
.to_string()
}
trait Indent {
fn indent(&self, level: usize) -> String;
}
impl Indent for &str {
fn indent(&self, level: usize) -> String {
let mut result = String::new();
for l in self.lines() {
if !l.is_empty() {
result.push_str(&" ".repeat(level));
result.push_str(l);
}
result.push('\n');
}
if !self.ends_with('\n') {
result.pop();
}
result
}
}
impl Indent for String {
fn indent(&self, level: usize) -> String {
self.as_str().indent(level)
}
}
trait Generator {
fn pattern(&self, level: usize) -> String;
fn error(&self, token_symbols: &FxHashMap<&str, &str>) -> String;
}
impl Generator for std::collections::BTreeSet<TokenName<'_>> {
fn pattern(&self, level: usize) -> String {
let symbols: Vec<_> = self.iter().map(|s| format!("Token::{}", s.0)).collect();
symbols.join(&format!("\n{}| ", " ".repeat(level)))
}
fn error(&self, token_symbols: &FxHashMap<&str, &str>) -> String {
let expected = self
.iter()
.filter_map(|s| token_symbols.get(s.0.as_ref()))
.map(|sym| {
if sym.starts_with('<') && sym.ends_with('>') && sym.len() > 2 {
sym.to_string()
} else {
format!("'{}'", sym)
}
})
.collect::<Vec<_>>();
syntax_error_message(&expected)
}
}
fn syntax_error_message(expected: &[String]) -> String {
let mut msg = if expected.is_empty() {
"invalid syntax"
} else if expected.len() == 1 {
"invalid syntax, expected: "
} else {
"invalid syntax, expected one of: "
}
.to_string();
msg.push_str(&expected.join(", "));
format!("\"{}\"", escape(&msg))
}
pub struct RustOutput {}
impl RustOutput {
pub fn run(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
input: &Path,
output: &Path,
) -> std::io::Result<()> {
let file = File::cast(cst, NodeRef::ROOT).unwrap();
let mut generated_file =
BufWriter::new(std::fs::File::create(output.join("generated.rs"))?);
generated_file.write_all(format!("// generated by lelwel {VERSION}\n\n").as_bytes())?;
Self::output_generated(cst, sema, file, &mut generated_file)?;
let parser_path = input.parent().unwrap().join("parser.rs");
let lexer_path = input.parent().unwrap().join("lexer.rs");
if !parser_path.exists() && !lexer_path.exists() {
Self::output_parser(sema, &parser_path)?;
Self::output_lexer(cst, file, &lexer_path)?;
}
Ok(())
}
fn output_parser(sema: &SemanticData<'_>, path: &Path) -> std::io::Result<()> {
let mut parser_file = BufWriter::new(std::fs::File::create(path)?);
let skeleton = include_str!("../skeleton/parser.rs");
parser_file.write_all(skeleton.as_bytes())?;
Self::output_parser_callbacks(&mut parser_file, sema, false, BTreeMap::default())
}
fn output_lexer(cst: &Cst<'_>, file: File, path: &Path) -> std::io::Result<()> {
let mut lexer_file = std::fs::File::create(path)?;
let mut token_enumerators = "{\n EOF,\n".to_string();
for token in file.token_decls(cst) {
if let Some((symbol, _)) = token.symbol(cst)
&& !(symbol.is_empty()
|| symbol.starts_with("'<") && symbol.ends_with(">'") && symbol.len() > 4)
{
token_enumerators += " #[token(\"";
token_enumerators += &symbol[1..symbol.len() - 1];
token_enumerators += "\")]\n";
}
let (name, _) = token.name(cst).unwrap();
token_enumerators += " ";
token_enumerators += name;
token_enumerators += ",\n";
}
let skeleton =
include_str!("../skeleton/lexer.rs").replace("{\n EOF,\n", &token_enumerators);
lexer_file.write_all(skeleton.as_bytes())
}
fn output_parser_callbacks(
output: &mut BufWriter<std::fs::File>,
sema: &SemanticData<'_>,
is_trait: bool,
rule_names: BTreeMap<&str, bool>,
) -> std::io::Result<()> {
output.write_all(if is_trait {
b"#[allow(clippy::ptr_arg)]\
\npub trait ParserCallbacks<'a> {\
\n type Diagnostic;\
\n type Context;\
\n\
\n /// Called at the start of the parse to generate all tokens and corresponding spans.\
\n fn create_tokens(context: &mut Self::Context, source: &'a str, diags: &mut Vec<Self::Diagnostic>) -> (Vec<Token>, Vec<Span>);\
\n /// Called when diagnostic is created.\
\n fn create_diagnostic(&self, span: Span, message: String) -> Self::Diagnostic;\
\n /// This predicate can be used to skip normal tokens.\
\n fn predicate_skip(&self, _token: Token) -> bool {\
\n false\
\n }\n\n"
} else {
b"impl<'a> ParserCallbacks<'a> for Parser<'a> {\
\n type Diagnostic = Diagnostic;\
\n type Context = (); // TODO: add context information to the parser if required\
\n\
\n fn create_tokens(_context: &mut Self::Context, source: &'a str, diags: &mut Vec<Self::Diagnostic>) -> (Vec<Token>, Vec<Span>) {\
\n tokenize(source, diags)\
\n }\
\n fn create_diagnostic(&self, span: Span, message: String) -> Self::Diagnostic {\
\n Self::Diagnostic::error()\
\n .with_message(message)\
\n .with_label(Label::primary((), span))\
\n }\n"
})?;
if is_trait {
for (rule_name, _) in rule_names.iter() {
output.write_all(
format!(
" /// Called when `{rule_name}` node is created.\
\n fn create_node_{rule_name}(&mut self, _node_ref: NodeRef, _diags: &mut Vec<Self::Diagnostic>) {{}}\n"
).as_bytes()
)?;
}
output.write_all(b"\n")?;
for (rule_name, in_choice) in rule_names {
if in_choice {
output.write_all(
format!(
" /// Called when `{rule_name}` node is deleted during backtracking.\
\n fn delete_node_{rule_name}(&mut self, _node_ref: NodeRef) {{}}\n"
)
.as_bytes(),
)?;
}
}
output.write_all(b"\n")?;
}
let mut predicates = FxHashSet::default();
for (rule, num) in sema.predicates.values() {
if predicates.contains(&(rule, num)) {
continue;
}
predicates.insert((rule, num));
if is_trait {
output.write_all(
format!(" /// Called when semantic predicate `?{num}` in rule `{rule}` is visited.\n")
.as_bytes(),
)?;
}
output.write_all(
format!(
" fn predicate_{rule}_{num}(&self) -> bool{}\n",
if is_trait {
";"
} else {
" {\n todo!()\n }"
}
)
.as_bytes(),
)?;
}
let mut actions = FxHashSet::default();
for (rule, num) in sema.actions.values() {
if actions.contains(&(rule, num)) {
continue;
}
actions.insert((rule, num));
if is_trait {
output.write_all(
format!(" /// Called when semantic action `#{num}` in rule `{rule}` is visited.\n")
.as_bytes(),
)?;
}
output.write_all(
format!(
" fn action_{rule}_{num}(&mut self, diags: &mut Vec<Self::Diagnostic>){}\n",
if is_trait {
";"
} else {
" {\n todo!()\n }"
}
)
.as_bytes(),
)?;
}
let mut assertions = FxHashSet::default();
for (rule, num) in sema.assertions.values() {
if assertions.contains(&(rule, num)) {
continue;
}
assertions.insert((rule, num));
if is_trait {
output.write_all(
format!(" /// Called when semantic assertion `!{num}` in rule `{rule}` is visited.\n")
.as_bytes(),
)?;
}
output.write_all(
format!(
" fn assertion_{rule}_{num}(&self) -> Option<Self::Diagnostic>{}\n",
if is_trait {
";"
} else {
" {\n todo!()\n }"
}
)
.as_bytes(),
)?;
}
output.write_all(b"}\n")
}
fn output_node_kind_decl(
output: &mut BufWriter<std::fs::File>,
has_rule_rename: bool,
name: &str,
level: usize,
is_decl: bool,
) -> std::io::Result<()> {
if has_rule_rename {
output.write_all(
format!(
"{}node_kind = Rule::{};\n",
if is_decl { "let mut " } else { "" },
snake_to_pascal_case(name),
)
.indent(level)
.as_bytes(),
)?;
}
Ok(())
}
fn output_cst_close(
output: &mut BufWriter<std::fs::File>,
has_rule_rename: bool,
name: &str,
level: usize,
assign_lhs: bool,
parser_name: &str,
is_start: bool,
) -> std::io::Result<()> {
let close = if is_start { "close_root" } else { "close" };
let lhs = if assign_lhs { "lhs = closed;" } else { "" };
if has_rule_rename {
output.write_all(
format!(
"let closed = {parser_name}.{close}(m, node_kind, diags);\
\n{parser_name}.create_node(node_kind, NodeRef(closed.0), diags);\
\n{lhs}\n"
)
.indent(level)
.as_bytes(),
)
} else {
output.write_all(
format!(
"let closed = {parser_name}.{close}(m, Rule::{}, diags);\
\n{parser_name}.create_node_{name}(NodeRef(closed.0), diags);\
\n{lhs}\n",
snake_to_pascal_case(name),
)
.indent(level)
.as_bytes(),
)
}
}
fn output_elision_init(
output: &mut BufWriter<std::fs::File>,
level: usize,
has_rule_creation: bool,
parser_name: &str,
is_start: bool,
elision: RuleNodeElision,
) -> std::io::Result<()> {
if !is_start {
match elision {
RuleNodeElision::None => output.write_all(
format!("let m = {parser_name}.open(diags);\n")
.indent(level)
.as_bytes(),
)?,
RuleNodeElision::Conditional => output.write_all(
format!("let start = {parser_name}.mark(diags);\nlet mut elide = false;\n")
.indent(level)
.as_bytes(),
)?,
RuleNodeElision::Unconditional => {
if has_rule_creation {
output.write_all(
format!("let start = {parser_name}.mark(diags);\n")
.indent(level)
.as_bytes(),
)?;
}
}
}
}
Ok(())
}
fn output_elision_check(
output: &mut BufWriter<std::fs::File>,
level: usize,
has_rule_rename: bool,
name: &str,
parser_name: &str,
is_start: bool,
elision: RuleNodeElision,
) -> std::io::Result<()> {
if !is_start {
match elision {
RuleNodeElision::None => {
Self::output_cst_close(
output,
has_rule_rename,
name,
level,
false,
parser_name,
is_start,
)?;
}
RuleNodeElision::Conditional => {
output.write_all(
format!("if !elide {{\nlet m = {parser_name}.open_before(start, diags);\n")
.indent(level)
.as_bytes(),
)?;
Self::output_cst_close(
output,
has_rule_rename,
name,
level + 1,
false,
parser_name,
is_start,
)?;
output.write_all("}\n".indent(level).as_bytes())?;
}
RuleNodeElision::Unconditional => {}
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn output_normal_rule(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
output: &mut BufWriter<std::fs::File>,
token_symbols: &FxHashMap<&str, &str>,
has_rule_rename: bool,
has_rule_creation: bool,
name: &str,
regex: Regex,
is_start: bool,
elision: RuleNodeElision,
) -> std::io::Result<()> {
Self::output_elision_init(output, 2, has_rule_creation, "self", is_start, elision)?;
if !is_start {
Self::output_node_kind_decl(output, has_rule_rename, name, 2, true)?;
}
Self::output_regex(
cst,
sema,
regex,
output,
2,
token_symbols,
false,
name,
elision,
"self",
has_rule_rename,
)?;
Self::output_elision_check(output, 2, has_rule_rename, name, "self", is_start, elision)
}
#[allow(clippy::too_many_arguments)]
fn output_left_recursive_rule(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
output: &mut BufWriter<std::fs::File>,
token_symbols: &FxHashMap<&str, &str>,
has_rule_rename: bool,
name: &str,
regex: Regex,
recursive: &RecursiveBranches,
in_choice: bool,
) -> std::io::Result<()> {
let requires_bp = recursive.branches().len() > 1
&& recursive
.branches()
.iter()
.any(|branch| matches!(branch, Recursion::Right(..) | Recursion::LeftRight(..)));
output.write_all(
format!(
" fn rec<'a>(\
\n parser: &mut Parser<'a>,\
\n diags: &mut Vec<<Parser<'a> as ParserCallbacks<'a>>::Diagnostic>,{}\
\n mut lhs: MarkClosed,\
\n ) {}{{\n",
if requires_bp {
"\n min_bp: usize,"
} else {
""
},
if in_choice { "-> Option<()> " } else { "" }
)
.as_bytes(),
)?;
let call_rec = |parser, binding_power, marker| {
if requires_bp {
format!(
"rec({parser}, diags, {binding_power}, {marker}){};\n",
if in_choice { "?" } else { "" }
)
} else {
format!(
"rec({parser}, diags, {marker}){};\n",
if in_choice { "?" } else { "" }
)
}
};
Self::output_node_kind_decl(output, has_rule_rename, name, 3, true)?;
output.write_all(b" match parser.current {\n")?;
let ops = if let Regex::Alternation(alt) = regex {
alt.operands(cst)
} else {
unreachable!();
};
let mut advance_error_set = BTreeSet::new();
for alt_op in ops {
let branch = recursive.get_branch(alt_op);
if !matches!(branch, None | Some(Recursion::Right(..))) {
continue;
}
let predict = &sema.predict_sets[&alt_op.syntax()];
let predicate = Self::get_predicate(cst, name, alt_op, "parser");
advance_error_set = if predicate.is_empty() {
advance_error_set.difference(predict).cloned().collect()
} else {
advance_error_set.union(predict).cloned().collect()
};
output.write_all(
format!("{}{predicate} => {{\n", predict.pattern(0),)
.indent(4)
.as_bytes(),
)?;
let elision = *sema.elision.get(&alt_op.syntax()).unwrap();
Self::output_elision_init(output, 5, false, "parser", false, elision)?;
if let Some(Recursion::Right(_, index)) = branch {
let binding_power = recursive.binding_power(alt_op).0;
let Regex::Concat(concat) = alt_op else {
unreachable!()
};
for (i, concat_op) in concat.operands(cst).enumerate() {
if i == index {
output.write_all(
format!(
"let lhs = parser.mark(diags);\n{}",
call_rec("parser", binding_power, "lhs")
)
.indent(5)
.as_bytes(),
)?;
} else {
Self::output_regex(
cst,
sema,
concat_op,
output,
5,
token_symbols,
false,
name,
elision,
"parser",
has_rule_rename,
)?;
}
}
} else {
Self::output_regex(
cst,
sema,
alt_op,
output,
5,
token_symbols,
false,
name,
elision,
"parser",
has_rule_rename,
)?;
}
Self::output_elision_check(output, 5, has_rule_rename, name, "parser", false, elision)?;
output.write_all("}\n".indent(4).as_bytes())?;
}
if !advance_error_set.is_empty() {
output.write_all(
format!(
" {} => {{\
\n parser.advance_with_error(diags, err![parser, {}]);\
\n }}\n",
advance_error_set.pattern(0),
syntax_error_message(&[])
)
.indent(3)
.as_bytes(),
)?;
}
output.write_all(
format!(
" _ => {{\
\n parser.error(diags, err![parser, {}]);\
\n }}\
\n}}\n",
sema.predict_sets[®ex.syntax()].error(token_symbols)
)
.indent(3)
.as_bytes(),
)?;
output.write_all(b" loop {\n")?;
Self::output_node_kind_decl(output, has_rule_rename, name, 4, false)?;
output.write_all(b" match parser.current {\n")?;
for branch in recursive.branches() {
let (concat, left_index, right_index) = match branch {
Recursion::Left(Regex::Concat(concat), index) => (concat, *index, None),
Recursion::LeftRight(Regex::Concat(concat), left_index, right_index) => {
(concat, *left_index, Some(*right_index))
}
_ => continue,
};
let binding_power = recursive.binding_power(branch.regex());
let mut is_first = true;
for (i, concat_op) in concat.operands(cst).enumerate() {
if let Regex::Predicate(..) = concat_op {
continue;
}
if i == left_index {
continue;
}
if is_first {
is_first = false;
output.write_all(
format!(
"{}{} => {{\n",
sema.predict_sets[&concat_op.syntax()].pattern(0),
Self::get_predicate(cst, name, branch.regex(), "parser")
)
.indent(5)
.as_bytes(),
)?;
if requires_bp {
let binding_power = binding_power.0;
output.write_all(
format!(
"if {binding_power} < min_bp {{\
\n break;\
\n}}\n"
)
.indent(6)
.as_bytes(),
)?;
}
output.write_all(
"let m = parser.open_before(lhs, diags);\n"
.indent(6)
.as_bytes(),
)?;
}
if right_index == Some(i) {
let binding_power = binding_power.1;
output.write_all(
format!(
"let rhs = parser.mark(diags);\n{}",
call_rec("parser", binding_power, "rhs")
)
.indent(6)
.as_bytes(),
)?;
} else {
Self::output_regex(
cst,
sema,
concat_op,
output,
6,
token_symbols,
false,
name,
RuleNodeElision::None,
"parser",
has_rule_rename,
)?;
}
}
Self::output_cst_close(output, has_rule_rename, name, 6, true, "parser", false)?;
output.write_all(b" continue;\n")?;
output.write_all(b" }\n")?;
}
output.write_all(
" _ => {\
\n break;\
\n }\
\n}\n"
.indent(4)
.as_bytes(),
)?;
output.write_all(
format!(
" }}{}\
\n}}\
\nlet lhs = self.mark(diags);\n{}",
if in_choice { "\n Some(())" } else { "" },
call_rec("self", 0, "lhs")
)
.indent(2)
.as_bytes(),
)
}
fn output_parts(
cst: &Cst<'_>,
rule: RuleDecl,
output: &mut BufWriter<std::fs::File>,
) -> std::io::Result<()> {
let name = rule.name(cst).unwrap().0;
output.write_all(
format!(
" /// Returns the CST for a parse of the {name} rule\
\n pub fn parse_{name}(mut self, diags: &mut Vec<Diagnostic>) -> Cst<'a> {{\
\n self.end_of_input = Token::EOF{};\
\n self.parse_rule(|parser, diags| parser.rule_{name}(diags), diags, Rule::Part)\
\n }}\n",
snake_to_pascal_case(name)
)
.as_bytes(),
)
}
fn output_rule(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
rule: RuleDecl,
output: &mut BufWriter<std::fs::File>,
token_symbols: &FxHashMap<&str, &str>,
) -> std::io::Result<()> {
if !sema.used.contains(&rule.syntax()) {
return Ok(());
}
let name = rule.name(cst).unwrap().0;
let recursive = sema.recursive.get(&rule);
let is_start = sema.start_rule.unwrap() == rule;
let has_rule_rename = sema.has_rule_rename.contains(&rule);
let has_rule_creation = sema.has_rule_creation.contains(&rule);
let elision = if rule.is_elided(cst) {
&RuleNodeElision::Unconditional
} else {
rule.regex(cst)
.and_then(|regex| sema.elision.get(®ex.syntax()))
.unwrap_or(&RuleNodeElision::None)
};
let in_choice = sema.used_in_ordered_choice.contains(&rule.syntax());
output.write_all(
format!(
" {}fn rule_{name}(&mut self, diags: &mut Vec<<Self as ParserCallbacks<'a>>::Diagnostic>) {}{{\n",
if has_rule_rename {
"#[allow(unused_assignments)]\n "
} else {
""
},
if in_choice { "-> Option<()> " } else { "" }
)
.as_bytes(),
)?;
if let Some(regex) = rule.regex(cst) {
if recursive.is_some_and(|recursive| {
recursive
.branches()
.iter()
.any(|rec| matches!(rec, Recursion::Left(..) | Recursion::LeftRight(..)))
}) {
Self::output_left_recursive_rule(
cst,
sema,
output,
token_symbols,
has_rule_rename,
name,
regex,
recursive.unwrap(),
in_choice,
)?;
} else {
Self::output_normal_rule(
cst,
sema,
output,
token_symbols,
has_rule_rename,
has_rule_creation,
name,
regex,
is_start,
*elision,
)?;
}
}
if in_choice {
output.write_all(b" Some(())\n")?;
}
output.write_all(b" }\n")?;
Ok(())
}
fn get_predicate(cst: &Cst<'_>, rule_name: &str, regex: Regex, parser_name: &str) -> String {
match regex {
Regex::Concat(concat) => match concat.operands(cst).next().unwrap() {
Regex::Predicate(pred) => {
if pred.is_true(cst) {
" if true".to_string()
} else {
format!(
" if {parser_name}.predicate_{rule_name}_{}()",
&pred.value(cst).unwrap().0[1..]
)
}
}
_ => "".to_string(),
},
Regex::Paren(paren) => {
if let Some(inner) = paren.inner(cst) {
Self::get_predicate(cst, rule_name, inner, parser_name)
} else {
"".to_string()
}
}
_ => "".to_string(),
}
}
#[allow(clippy::too_many_arguments)]
fn output_recovering_operation(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
regex: Regex,
output: &mut BufWriter<std::fs::File>,
level: usize,
token_symbols: &FxHashMap<&str, &str>,
open_before: bool,
rule_name: &str,
rule_elision: RuleNodeElision,
parser_name: &str,
has_rule_rename: bool,
op: Regex,
ordered_choice_return: &str,
is_loop: bool,
) -> std::io::Result<()> {
output.write_all(
format!(
"loop {{\
\n match {parser_name}.current {{\
\n {}{} => {{\n",
sema.first_sets[&op.syntax()].pattern(2),
Self::get_predicate(cst, rule_name, op, parser_name)
)
.indent(level)
.as_bytes(),
)?;
if open_before {
output.write_all(
format!(
"if m.is_none() {{\
\n m = Some({parser_name}.open_before(lhs, diags));\
\n}}\n"
)
.indent(5)
.as_bytes(),
)?;
}
Self::output_regex(
cst,
sema,
op,
output,
level + 3,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
let ordered_choice_return = ordered_choice_return.indent(3);
let follow = sema.follow_sets[®ex.syntax()].pattern(2);
let expected = if is_loop {
sema.follow_sets[&op.syntax()].error(token_symbols)
} else {
sema.predict_sets[®ex.syntax()].error(token_symbols)
};
let recovery = &sema.recovery_sets[®ex.syntax()];
let recovery = if recovery.is_empty() {
"".to_string()
} else {
format!(
"\n | {} => {{{ordered_choice_return}\
\n {parser_name}.error(diags, err![{parser_name}, {expected}]);\
\n break;\
\n }}",
recovery.pattern(2)
)
};
if !is_loop {
output.write_all(" break;\n".indent(level).as_bytes())?;
}
output.write_all(
format!(
" }}\
\n {follow} => break,{recovery}\
\n _ => {{{ordered_choice_return}\
\n {parser_name}.advance_with_error(diags, err![{parser_name}, {expected}]);\
\n }}\
\n }}\
\n}}\n",
)
.indent(level)
.as_bytes(),
)
}
#[allow(clippy::too_many_arguments)]
fn output_regex(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
regex: Regex,
output: &mut BufWriter<std::fs::File>,
level: usize,
token_symbols: &FxHashMap<&str, &str>,
open_before: bool,
rule_name: &str,
rule_elision: RuleNodeElision,
parser_name: &str,
has_rule_rename: bool,
) -> std::io::Result<()> {
let in_choice = sema.used_in_ordered_choice.contains(®ex.syntax());
let expect_macro = if in_choice { "try_expect!" } else { "expect!" };
let ordered_choice_return = if in_choice {
format!(
"\nif {parser_name}.in_ordered_choice {{\
\n return None;\
\n}}"
)
} else {
"".to_string()
};
match regex {
Regex::Name(name) => {
let decl = sema.decl_bindings[&name.syntax()];
if let Some(rule) = RuleDecl::cast(cst, decl) {
let name = rule.name(cst).unwrap().0;
let rule_in_choice = sema.used_in_ordered_choice.contains(&rule.syntax());
output.write_all(
format!(
"{parser_name}.rule_{name}(diags){};\n",
if rule_in_choice && in_choice { "?" } else { "" }
)
.indent(level)
.as_bytes(),
)?;
} else if let Some(token) = TokenDecl::cast(cst, decl) {
let name = token.name(cst).unwrap().0;
let sym = token
.symbol(cst)
.map_or(name, |(sym, _)| &sym[1..sym.len() - 1]);
output.write_all(
format!(
"{expect_macro}({name}, {}, {parser_name}, diags);\n",
syntax_error_message(&[sym.to_string()])
)
.indent(level)
.as_bytes(),
)?;
}
}
Regex::Symbol(sym) => {
let decl = sema.decl_bindings[&sym.syntax()];
if let Some(token) = TokenDecl::cast(cst, decl) {
let name = token.name(cst).unwrap().0;
let sym = token.symbol(cst).unwrap().0;
output.write_all(
format!(
"{expect_macro}({name}, {}, {parser_name}, diags);\n",
syntax_error_message(&[sym.to_string()])
)
.indent(level)
.as_bytes(),
)?;
}
}
Regex::OrderedChoice(choice) => {
output.write_all(
format!("{parser_name}.in_ordered_choice = true;\n")
.indent(level)
.as_bytes(),
)?;
output.write_all("'ordered_choice: {\n".indent(level).as_bytes())?;
output.write_all(
format!("let state = {parser_name}.get_state(diags);\n")
.indent(level + 1)
.as_bytes(),
)?;
if rule_elision == RuleNodeElision::Conditional {
output
.write_all("let elision_state = elide;\n".indent(level + 1).as_bytes())?;
}
if has_rule_rename {
output.write_all(
"let node_kind_state = node_kind;\n"
.indent(level + 1)
.as_bytes(),
)?;
}
let ops = choice.operands(cst).collect::<Vec<_>>();
for op in &ops[..ops.len() - 1] {
let predict = &sema.predict_sets[&op.syntax()];
output.write_all(
format!(
"if matches!({parser_name}.current, {}) {{\n",
predict.pattern(1)
)
.indent(level + 1)
.as_bytes(),
)?;
output.write_all("if (|| {\n".indent(level + 2).as_bytes())?;
Self::output_regex(
cst,
sema,
*op,
output,
level + 3,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
output.write_all(
" Some(())\
\n})().is_some() {\
\n break 'ordered_choice;\
\n}\n"
.indent(level + 2)
.as_bytes(),
)?;
output.write_all(
format!("{parser_name}.set_state(&state, diags);\n")
.indent(level + 2)
.as_bytes(),
)?;
if rule_elision == RuleNodeElision::Conditional {
output
.write_all("elide = elision_state;\n".indent(level + 2).as_bytes())?;
}
if has_rule_rename {
output.write_all(
"node_kind = node_kind_state;\n"
.indent(level + 2)
.as_bytes(),
)?;
}
output.write_all("}\n".indent(level + 1).as_bytes())?;
}
output.write_all(
format!("{parser_name}.in_ordered_choice = false;\n")
.indent(level + 1)
.as_bytes(),
)?;
let op = *ops.last().unwrap();
let predict = &sema.predict_sets[&op.syntax()];
output.write_all(
format!(
"if matches!({parser_name}.current, {}) {{\n",
predict.pattern(1)
)
.indent(level + 1)
.as_bytes(),
)?;
Self::output_regex(
cst,
sema,
op,
output,
level + 2,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
output.write_all("} else {\n".indent(level + 1).as_bytes())?;
output.write_all(
format!(
"{parser_name}.advance_with_error(diags, err![{parser_name}, {}]);\n",
syntax_error_message(&[])
)
.indent(level + 2)
.as_bytes(),
)?;
output.write_all("}\n".indent(level + 1).as_bytes())?;
output.write_all("}\n".indent(level).as_bytes())?;
}
Regex::Concat(concat) => {
for op in concat.operands(cst) {
Self::output_regex(
cst,
sema,
op,
output,
level,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
}
}
Regex::Alternation(alt) => {
output.write_all(
format!("match {parser_name}.current {{\n")
.indent(level)
.as_bytes(),
)?;
let mut advance_error_set = BTreeSet::new();
for op in alt.operands(cst) {
let predict = &sema.predict_sets[&op.syntax()];
let predicate = Self::get_predicate(cst, rule_name, op, parser_name);
advance_error_set = if predicate.is_empty() {
advance_error_set.difference(predict).cloned().collect()
} else {
advance_error_set.union(predict).cloned().collect()
};
output.write_all(
format!("{}{predicate} => {{\n", predict.pattern(0))
.indent(level + 1)
.as_bytes(),
)?;
Self::output_regex(
cst,
sema,
op,
output,
level + 2,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
output.write_all("}\n".indent(level + 1).as_bytes())?;
}
if !advance_error_set.is_empty() {
output.write_all(
format!(
" {} => {{{}\
\n {parser_name}.advance_with_error(diags, err![{parser_name}, {}]);\
\n }}\n",
advance_error_set.pattern(0),
ordered_choice_return.indent(2),
syntax_error_message(&[])
)
.indent(level)
.as_bytes(),
)?;
}
output.write_all(
format!(
" _ => {{{}\
\n {parser_name}.error(diags, err![{parser_name}, {}]);\
\n }}\
\n}}\n",
ordered_choice_return.indent(2),
sema.predict_sets[®ex.syntax()].error(token_symbols),
)
.indent(level)
.as_bytes(),
)?;
}
Regex::Star(star) => {
Self::output_recovering_operation(
cst,
sema,
regex,
output,
level,
token_symbols,
open_before,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
star.operand(cst).unwrap(),
&ordered_choice_return,
true,
)?;
}
Regex::Plus(plus) => {
let op = plus.operand(cst).unwrap();
Self::output_regex(
cst,
sema,
op,
output,
level,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
Self::output_recovering_operation(
cst,
sema,
regex,
output,
level,
token_symbols,
open_before,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
op,
&ordered_choice_return,
true,
)?;
}
Regex::Optional(opt) => {
Self::output_recovering_operation(
cst,
sema,
regex,
output,
level,
token_symbols,
open_before,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
opt.operand(cst).unwrap(),
&ordered_choice_return,
false,
)?;
}
Regex::Paren(paren) => {
if let Some(inner) = paren.inner(cst) {
Self::output_regex(
cst,
sema,
inner,
output,
level,
token_symbols,
false,
rule_name,
rule_elision,
parser_name,
has_rule_rename,
)?;
}
}
Regex::Action(action) => {
output.write_all(
format!(
"{parser_name}.action_{rule_name}_{}(diags);\n",
&action.value(cst).unwrap().0[1..]
)
.indent(level)
.as_bytes(),
)?;
}
Regex::Assertion(assertion) => {
output.write_all(
format!(
"if let Some(diag) = {parser_name}.assertion_{rule_name}_{}() {{{}\
\n {parser_name}.error_since_advance = true;\
\n diags.push(diag);\
\n}}\n",
&assertion.value(cst).unwrap().0[1..],
ordered_choice_return.indent(1),
)
.indent(level)
.as_bytes(),
)?;
}
Regex::NodeRename(rename) => {
let name = &rename.value(cst).unwrap().0[1..];
if !name.is_empty() {
output.write_all(
format!("node_kind = Rule::{};\n", snake_to_pascal_case(name))
.indent(level)
.as_bytes(),
)?;
}
}
Regex::NodeElision(_) => {
if rule_elision == RuleNodeElision::Conditional {
output.write_all("elide = true;\n".indent(level).as_bytes())?;
}
}
Regex::NodeMarker(marker) => {
let number = marker.number(cst);
output.write_all(
format!("let m{number} = {parser_name}.mark(diags);\n")
.indent(level)
.as_bytes(),
)?;
}
Regex::NodeCreation(creation) => {
let node_name = creation.node_name(cst).unwrap_or(rule_name);
let mark = if creation.whole_rule(cst) {
"start".to_string()
} else {
format!("m{}", creation.number(cst).unwrap())
};
output.write_all(
format!(
"let open_node = {parser_name}.open_before({mark}, diags);\
\n{parser_name}.close(open_node, Rule::{}, diags);\
\n{parser_name}.create_node_{node_name}(NodeRef({mark}.0), diags);\n",
snake_to_pascal_case(node_name)
)
.indent(level)
.as_bytes(),
)?;
}
Regex::Commit(_) => {
output.write_all(
format!("{parser_name}.in_ordered_choice = false;\n")
.indent(level)
.as_bytes(),
)?;
}
Regex::Return(_) => {
output.write_all(
format!("if {parser_name}.active_error() {{\n")
.indent(level)
.as_bytes(),
)?;
match rule_elision {
RuleNodeElision::None => {
output.write_all(
format!(
"let closed = {parser_name}.close(m, Rule::Error, diags);\
\n{parser_name}.create_node_error(NodeRef(closed.0), diags);\n"
)
.indent(level + 1)
.as_bytes(),
)?;
}
RuleNodeElision::Conditional => {
output.write_all(
format!(
"if !elide {{\
\n let m = {parser_name}.open_before(start, diags);\
\n let closed = {parser_name}.close(m, Rule::Error, diags);\
\n {parser_name}.create_node_error(NodeRef(closed.0), diags);\
\n}}\n"
)
.indent(level + 1)
.as_bytes(),
)?;
}
RuleNodeElision::Unconditional => {}
}
output.write_all(
format!(
" return{};\
\n}}\n",
if in_choice { " None" } else { "" }
)
.indent(level)
.as_bytes(),
)?;
}
Regex::Predicate(_) => {}
}
Ok(())
}
fn output_generated(
cst: &Cst<'_>,
sema: &SemanticData<'_>,
file: File,
output: &mut BufWriter<std::fs::File>,
) -> std::io::Result<()> {
let mut token_symbols = FxHashMap::from_iter([("EOF", "<end of file>")]);
for token in file.token_decls(cst) {
let name = token.name(cst).unwrap().0;
let sym = token
.symbol(cst)
.map_or(name, |(sym, _)| &sym[1..sym.len() - 1]);
token_symbols.insert(name, sym);
}
let contains_ordered_choice = !sema.used_in_ordered_choice.is_empty();
let mut rule_names = BTreeMap::from_iter([("error", contains_ordered_choice)]);
if !sema.parts.is_empty() {
rule_names.insert("part", false);
}
for rule in file.rule_decls(cst) {
let in_choice = sema.used_in_ordered_choice.contains(&rule.syntax());
rule_names
.entry(rule.name(cst).unwrap().0)
.and_modify(|val| *val |= in_choice)
.or_insert(in_choice);
}
for (rule_name, node_refs) in sema.rule_bindings.iter() {
let in_choice = node_refs.iter().fold(false, |acc, node_ref| {
acc | sema.used_in_ordered_choice.contains(node_ref)
});
rule_names
.entry(rule_name)
.and_modify(|val| *val |= in_choice)
.or_insert(in_choice);
}
let mut rules = "".to_string();
let mut rules_fmt = "".to_string();
let mut rules_create = "".to_string();
let mut rules_delete = if contains_ordered_choice {
"match _rule {".to_string()
} else {
"".to_string()
};
let mut delete_covered = true;
for (rule_name, in_choice) in rule_names.iter() {
let pascal_case_name = &snake_to_pascal_case(rule_name);
rules += "\n ";
rules += pascal_case_name;
rules += ",";
rules_fmt += "\n Rule::";
rules_fmt += pascal_case_name;
rules_fmt += " => write!(f, \"";
rules_fmt += rule_name;
rules_fmt += "\"),";
rules_create += "\n Rule::";
rules_create += pascal_case_name;
rules_create += " => self.create_node_";
rules_create += rule_name;
rules_create += "(node_ref, diags),";
delete_covered &= *in_choice;
if *in_choice {
rules_delete += "\n Rule::";
rules_delete += pascal_case_name;
rules_delete += " => self.delete_node_";
rules_delete += rule_name;
rules_delete += "(_node_ref),";
}
}
if contains_ordered_choice {
if !delete_covered {
rules_delete += "\n _ => {}";
}
rules_delete += "\n }"
}
let mut skip = "".to_string();
for token in sema.skipped.iter() {
skip += " | Token::";
skip += token.name(cst).unwrap().0;
}
let start_rule = sema.start_rule.unwrap().name(cst).unwrap().0;
let start_rule_pascal_case = snake_to_pascal_case(start_rule);
output.write_all(
format!(
include_str!("../skeleton/generated.rs"),
rules,
skip,
start_rule,
start_rule_pascal_case,
rules_fmt,
rules_create,
rules_delete,
)
.as_bytes(),
)?;
for rule in sema.parts.iter() {
Self::output_parts(cst, *rule, output)?;
}
for rule in file.rule_decls(cst) {
Self::output_rule(cst, sema, rule, output, &token_symbols)?;
}
output.write_all(b"}\n\n")?;
Self::output_parser_callbacks(output, sema, true, rule_names)
}
}