#[cfg(test)]
pub(crate) fn render_parser(
grammar_name: &str,
data: &ParserCodegenData<'_>,
) -> io::Result<String> {
render_parser_with_options(grammar_name, data, ParserRenderOptions::default())
}
#[derive(Debug)]
pub(crate) struct ParserRenderModel {
pub(crate) support_bindings: GeneratedSupportBindings,
pub(crate) embedded_imports: &'static str,
pub(crate) token_constants: String,
pub(crate) rule_constants: String,
pub(crate) metadata: String,
pub(crate) parser_semantics_function: String,
pub(crate) typed_hook_adapter: String,
pub(crate) embedded_attrs_structs: String,
pub(crate) embedded_module_items: String,
pub(crate) parser_atn_data: String,
pub(crate) parse_convenience: String,
pub(crate) parser_rustdoc: String,
pub(crate) type_name: String,
pub(crate) adaptive_atn_preference_struct_field: String,
pub(crate) generated_rule_error: String,
pub(crate) base_initialization: String,
pub(crate) adaptive_atn_preference_field_init: String,
pub(crate) embedded_struct_fields: String,
pub(crate) embedded_field_inits: String,
pub(crate) parse_listener_facade: &'static str,
pub(crate) adaptive_atn_preference_reset: &'static str,
pub(crate) compile_pattern_method: &'static str,
pub(crate) adaptive_direct_allowed: bool,
pub(crate) parse_rule_fallback: String,
pub(crate) generated_rule_dispatch: String,
pub(crate) embedded_impl_items: String,
pub(crate) rule_methods: String,
pub(crate) action_method: String,
pub(crate) typed_parser_constructor: String,
}
const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[
"reset",
"set_token_stream",
"token_stream",
"token_stream_mut",
"token_store",
"parse_tree_storage",
"clear_dfa",
"add_error_listener",
"remove_error_listeners",
"add_parse_listener",
"remove_parse_listeners",
"node",
"into_token_stream",
"into_token_store",
"into_parsed_file",
"compile_parse_tree_pattern",
];
pub(crate) fn parser_public_rule_method_names(rule_names: &[String]) -> Vec<String> {
let mut used = GENERATED_PARSER_RESERVED_RULE_METHODS
.iter()
.map(|name| (*name).to_owned())
.collect::<BTreeSet<_>>();
rule_names
.iter()
.map(|rule| {
let base = rust_function_name(rule);
let name = unique_rule_method_name(&base, &used);
used.insert(name.clone());
name
})
.collect()
}
fn unique_rule_method_name(base: &str, used: &BTreeSet<String>) -> String {
if !used.contains(base) {
return base.to_owned();
}
let plain = base.strip_prefix("r#").unwrap_or(base);
let reserved_collision = GENERATED_PARSER_RESERVED_RULE_METHODS.contains(&base);
let stem = if reserved_collision {
format!("{plain}_rule")
} else {
plain.to_owned()
};
let (mut candidate, mut suffix) = if reserved_collision {
(stem.clone(), 2)
} else {
(format!("{stem}_2"), 3)
};
while used.contains(&candidate) {
candidate = format!("{stem}_{suffix}");
suffix += 1;
}
candidate
}
#[derive(Debug, Default)]
pub(crate) struct PortableLocalData {
pub(crate) declarations: Vec<Vec<String>>,
pub(crate) inline_actions: BTreeMap<usize, String>,
pub(crate) predicates: BTreeMap<(usize, usize), (String, Option<String>)>,
pub(crate) required_generated_rules: BTreeSet<usize>,
}
impl PortableLocalData {
fn has_semantics(&self) -> bool {
!self.required_generated_rules.is_empty()
}
fn step_render(&self) -> Option<PortableLocalStepRender<'_>> {
self.has_semantics().then_some(PortableLocalStepRender {
declarations: &self.declarations,
predicates: &self.predicates,
required_generated_rules: &self.required_generated_rules,
})
}
}
fn portable_local_name(name: &str) -> String {
format!("__antlr_local_{name}")
}
fn parse_portable_bool_assignment(body: &str) -> Option<(&str, bool)> {
let body = body.trim();
let body = body.strip_suffix(';').unwrap_or(body);
let (target, value) = body.split_once('=')?;
let target = target.trim().strip_prefix('$')?;
let value = match value.trim() {
"true" => true,
"false" => false,
_ => return None,
};
(!target.is_empty()
&& target
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()))
.then_some((target, value))
}
fn parse_portable_bool_predicate(body: &str) -> Option<(&str, bool)> {
let body = body.trim();
let (body, negated) = body
.strip_prefix('!')
.map_or((body, false), |body| (body.trim(), true));
let name = body.strip_prefix('$')?.trim();
(!name.is_empty()
&& name
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()))
.then_some((name, negated))
}
pub(crate) fn build_structural_portable_local_data(
data: &RecognizerCodegenData<'_>,
patterns: &SemPatternFile,
) -> io::Result<PortableLocalData> {
let model = structural_embedded_model(data, false)?;
let mut out = PortableLocalData {
declarations: vec![Vec::new(); data.rule_names.len()],
..PortableLocalData::default()
};
let mut local_names = Vec::with_capacity(data.rule_names.len());
for (rule_index, rule) in model.rules.iter().enumerate() {
let mut names = BTreeMap::new();
for attr in rule.attrs.iter().filter(|attr| attr.ty == "bool") {
let initial = if rule.local_names.contains(&attr.name) {
"false"
} else if rule.arg_names.first() == Some(&attr.name) {
"__precedence != 0"
} else {
continue;
};
let local = portable_local_name(&attr.name);
out.declarations[rule_index].push(format!("let mut {local} = {initial};"));
names.insert(attr.name.clone(), local);
}
local_names.push(names);
}
for action in structural_actions(data)? {
let Some((name, value)) = parse_portable_bool_assignment(&action.body) else {
continue;
};
let Some(local) = local_names
.get(action.rule_index)
.and_then(|names| names.get(name))
else {
continue;
};
if patterns
.coordinate_disposition(
SemanticsKind::ParserAction,
data.rule_names.get(action.rule_index).map(String::as_str),
Some(action.action_index),
Some(action.state),
)
.is_some()
{
continue;
}
out.inline_actions
.insert(action.state, format!("{local} = {value};"));
out.required_generated_rules.insert(action.rule_index);
}
for predicate in structural_predicates(data)? {
let Some((name, negated)) = parse_portable_bool_predicate(&predicate.body) else {
continue;
};
let Some(local) = local_names
.get(predicate.rule_index)
.and_then(|names| names.get(name))
else {
continue;
};
if patterns
.coordinate_disposition(
SemanticsKind::ParserPredicate,
data.rule_names
.get(predicate.rule_index)
.map(String::as_str),
Some(predicate.predicate_index),
None,
)
.is_some()
{
continue;
}
let expression = if negated {
format!("!{local}")
} else {
local.clone()
};
out.predicates.insert(
(predicate.rule_index, predicate.predicate_index),
(expression, predicate.fail),
);
out.required_generated_rules.insert(predicate.rule_index);
}
Ok(out)
}
fn collect_noop_action_states(
data: &ParserCodegenData<'_>,
patterns: &SemPatternFile,
) -> BTreeSet<usize> {
let mut noop_action_states = BTreeSet::new();
let action_state_coordinates = parser_action_state_coordinates(data);
for state in action_state_coordinates.keys() {
if parser_action_assume_overridden(patterns, data, &action_state_coordinates, *state) {
noop_action_states.insert(*state);
}
}
noop_action_states
}
fn render_public_rule_methods(public_rule_method_names: &[String]) -> String {
let mut rule_methods = String::new();
for (index, rule_method_name) in public_rule_method_names.iter().enumerate() {
writeln!(
rule_methods,
" pub fn {rule_method_name}(&mut self) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
writeln!(rule_methods, " self.parse_rule({index})")
.expect("writing to a string cannot fail");
writeln!(rule_methods, " }}").expect("writing to a string cannot fail");
}
rule_methods
}
fn embedded_step_render<'a>(
embedded: &'a ParserSurfaceBindings,
decisions: &'a ParserDecisionAnalysis,
) -> EmbeddedStepRender<'a> {
EmbeddedStepRender {
force_adaptive: false,
adaptive_decisions: &decisions.adaptive_decisions,
complete_ll1_dispatches: &decisions.complete_ll1_dispatches,
predicates: &embedded.predicates,
rule_has_attrs: &embedded.rule_has_attrs,
init_entry: &embedded.init_entry,
after: &embedded.after,
call_args: &embedded.call_args,
rule_arg0: &embedded.rule_arg0,
}
}
fn embedded_render_slots(
embedded_data: Option<&ParserSurfaceBindings>,
) -> (String, String, String, String, String) {
embedded_data.map_or_else(
|| {
(
String::new(),
String::new(),
String::new(),
String::new(),
String::new(),
)
},
|embedded| {
(
embedded.attrs_structs.clone(),
embedded.module_items.clone(),
embedded.struct_fields.clone(),
embedded.field_inits.clone(),
embedded.impl_items.clone(),
)
},
)
}
const fn render_parse_listener_facade() -> &'static str {
r"
/// Registers a listener for committed rule enter/exit events during
/// recognition (ANTLR's `addParseListener`). See
/// [`antlr4_runtime::ParseListener`] for the delivery contract.
pub fn add_parse_listener<T>(&mut self, listener: T)
where
T: antlr4_runtime::ParseListener + 'static,
{
self.base.add_parse_listener(listener);
}
/// Removes every registered parse listener and returns them, dropping
/// any sticky abort a removed listener had requested.
pub fn remove_parse_listeners(&mut self) -> Vec<Box<dyn antlr4_runtime::ParseListener>> {
self.base.remove_parse_listeners()
}
"
}
const fn render_compile_parse_tree_pattern_method() -> &'static str {
r#"
/// Compiles a tree pattern rooted at parser rule `rule_index`.
///
/// Mirrors ANTLR's `Parser.compileParseTreePattern`. Literal chunks of
/// `pattern` are lexed with a fresh lexer built by `make_lexer` (pass this
/// grammar's generated lexer constructor, e.g. `MyGrammarLexer::new`);
/// `<tag>` placeholders become rule/token references matched over a
/// rule-bypass ATN. The returned [`antlr4_runtime::ParseTreePattern`] can
/// then match subtrees.
///
/// Takes `&self` only to mirror ANTLR's instance method; the ATN and
/// grammar metadata come from this module, so the parser's own state is
/// untouched. The pattern compiler (and its rule-bypass ATN) is built once
/// per process and shared by every call.
///
/// # Errors
///
/// Returns a [`antlr4_runtime::ParseTreePatternError`] for a malformed
/// pattern, an unknown tag, a lexer failure, or a pattern the start rule
/// does not parse cleanly and fully consume.
pub fn compile_parse_tree_pattern<PL>(
&self,
pattern: &str,
rule_index: usize,
mut make_lexer: impl FnMut(antlr4_runtime::InputStream) -> PL,
) -> Result<antlr4_runtime::ParseTreePattern, antlr4_runtime::ParseTreePatternError>
where
PL: antlr4_runtime::TokenSource,
{
// The rule-bypass ATN derivation inside `ParseTreePatternMatcher::new`
// is O(states + transitions), so — like ANTLR's
// `Parser.bypassAltsAtnCache` — the matcher is built once per process
// and shared by every subsequent compile. A failed build is not cached
// and is retried (and re-reported) on the next call.
static PATTERN_DATA: OnceLock<RecognizerData> = OnceLock::new();
static PATTERN_MATCHER: OnceLock<antlr4_runtime::ParseTreePatternMatcher<'static>> =
OnceLock::new();
let matcher = match PATTERN_MATCHER.get() {
Some(matcher) => matcher,
None => {
let data = PATTERN_DATA.get_or_init(|| {
let grammar_metadata = metadata();
grammar_metadata.recognizer_data()
});
let matcher = antlr4_runtime::ParseTreePatternMatcher::new(parser_atn(), data)?;
PATTERN_MATCHER.get_or_init(|| matcher)
}
};
matcher.compile(pattern, rule_index, move |text: &str| {
antlr4_runtime::lex_pattern_chunk(text, &mut make_lexer)
})
}
"#
}
fn decision_routing_render<'a>(
classification: &'a ParserDecisionAnalysis,
options: ParserRenderOptions<'_>,
) -> DecisionRoutingRender<'a> {
DecisionRoutingRender {
complete_ll1_dispatches: (!options.embedded)
.then_some(&classification.complete_ll1_dispatches),
ll1_dispatch_tables: (!options.embedded && options.fixed_lookahead.is_some())
.then_some(&classification.ll1_dispatch_tables),
fixed_lookahead_tables: options
.fixed_lookahead
.is_some_and(|depth| depth >= 2)
.then_some(&classification.fixed_lookahead_tables),
}
}
fn build_parser_surface_model(
data: &RecognizerCodegenData<'_>,
type_name: &str,
grammar_name: &str,
options: ParserRenderOptions<'_>,
) -> io::Result<ParserSurfaceModel> {
if options.embedded {
let embedded = build_embedded_parser_data(data, type_name, grammar_name, options)?;
Ok(ParserSurfaceModel::embedded(embedded))
} else {
let surface = build_structural_parser_surface(data, grammar_name, options)?;
Ok(ParserSurfaceModel::structural(surface))
}
}
struct ParserActionRouting {
inline_statements: BTreeMap<usize, String>,
states: BTreeSet<usize>,
generated_states: BTreeSet<usize>,
indices: BTreeMap<usize, usize>,
committed_indices: Vec<(usize, usize)>,
}
fn parser_action_routing(
data: &ParserCodegenData<'_>,
embedded: bool,
embedded_data: Option<&ParserSurfaceBindings>,
portable_local_data: &PortableLocalData,
parameterized_rules: &BTreeSet<usize>,
noop_states: &BTreeSet<usize>,
) -> io::Result<ParserActionRouting> {
let mut inline_statements = embedded_data.map_or_else(
|| portable_local_data.inline_actions.clone(),
|embedded| embedded.inline_actions.clone(),
);
let states = parser_action_states(data)
.into_iter()
.collect::<BTreeSet<_>>();
let structural_actions = structural_actions(data)?;
let indices = structural_actions
.iter()
.map(|action| (action.state, action.action_index))
.collect::<BTreeMap<_, _>>();
let action_rules = structural_actions
.iter()
.map(|action| (action.state, action.rule_index))
.collect::<BTreeMap<_, _>>();
let committed_indices = structural_actions
.iter()
.filter(|action| {
!embedded
&& action.authored
&& !action.body.trim().is_empty()
&& !noop_states.contains(&action.state)
})
.map(|action| (action.state, action.action_index))
.collect::<Vec<_>>();
let mut generated_states = if embedded {
states.clone()
} else {
noop_states.intersection(&states).copied().collect()
};
generated_states.extend(portable_local_data.inline_actions.keys().copied());
if !embedded {
for state in states
.difference(noop_states)
.filter(|state| !portable_local_data.inline_actions.contains_key(state))
{
let statement = if action_rules
.get(state)
.is_some_and(|rule_index| parameterized_rules.contains(rule_index))
{
"let _ = self.base.parser_action_hook_with_context_and_local(action, &__ctx, __precedence);"
} else {
"let _ = self.base.parser_action_hook_with_context(action, &__ctx);"
};
inline_statements.insert(*state, statement.to_owned());
generated_states.insert(*state);
}
}
Ok(ParserActionRouting {
inline_statements,
states,
generated_states,
indices,
committed_indices,
})
}
fn render_parser_action_index_array(action_indices: &[(usize, usize)]) -> String {
let items = action_indices
.iter()
.map(|(source_state, action_index)| format!("({source_state}, {action_index})"))
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
fn render_parser_rule_arg_array(args: &[(usize, usize, RuleArgTemplate)]) -> String {
let items = args
.iter()
.map(|(source_state, rule_index, value)| {
let (value, inherit_local) = match value {
RuleArgTemplate::Literal(value) => (*value, false),
RuleArgTemplate::InheritLocal => (0, true),
};
format!(
"antlr4_runtime::ParserRuleArg {{ source_state: {source_state}, rule_index: {rule_index}, value: {value}, inherit_local: {inherit_local} }}"
)
})
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
fn render_parser_base_initialization(
unknown_policy_literal: Option<&str>,
member_seeds: &str,
) -> String {
let needs_mut = unknown_policy_literal.is_some() || !member_seeds.is_empty();
let mut out = if needs_mut {
" let mut base = BaseParser::with_semantic_hooks(input, data, hooks);".to_owned()
} else {
" let base = BaseParser::with_semantic_hooks(input, data, hooks);".to_owned()
};
if let Some(policy) = unknown_policy_literal {
write!(
out,
"\n base.set_unknown_predicate_policy({policy});"
)
.expect("writing to a string cannot fail");
}
if !member_seeds.is_empty() {
write!(out, "\n base.set_initial_members({member_seeds});")
.expect("writing to a string cannot fail");
}
out
}
pub(crate) fn render_parser_parse_convenience(type_name: &str, surface_name: &str) -> String {
let output_type_name = format!("{type_name}ParseOutput");
let validated_tree_name = format!("{surface_name}ValidatedTree");
let validation_error_name = format!("{surface_name}ValidationError");
format!(
r#"/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`].
///
/// Keeps the generated parser available after the entry rule runs so callers
/// can inspect diagnostics or recover the parser-owned token stream.
#[derive(Debug)]
pub struct {output_type_name}<R, L>
where
L: TokenSource,
{{
pub result: R,
pub parser: {type_name}<L>,
}}
impl<L: TokenSource> {output_type_name}<antlr4_runtime::NodeId, L> {{
/// Validates a completed parse and changes its generated context surface.
///
/// Validation rejects lexer diagnostics, parser recovery, recovered error
/// nodes, and missing generated required children before constructing the
/// validated-tree type boundary.
pub fn validate(self) -> Result<{validated_tree_name}, {validation_error_name}> {{
let lexer = self.parser.token_stream().number_of_source_errors();
let parser = self.parser.number_of_syntax_errors();
if lexer != 0 || parser != 0 {{
return Err({validation_error_name}::SyntaxErrors {{ lexer, parser }});
}}
let parsed = self.parser.into_parsed_file(self.result);
validate_tree_structure(&parsed)?;
Ok({validated_tree_name}::__new(parsed))
}}
}}
/// Parses UTF-8 text by constructing the lexer, token stream, parser, and
/// caller-selected entry rule in one call.
///
/// Pass the generated lexer constructor and a parser entry rule, for example
/// `parse(src, MyGrammarLexer::new, {type_name}::file)`.
///
/// The returned [`antlr4_runtime::ParsedFile`] owns the canonical token store,
/// flat CST storage, and entry-rule root.
/// Use [`parse_with_parser`] instead when the caller also needs parser
/// diagnostics after the entry rule runs.
pub fn parse<L: TokenSource>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<antlr4_runtime::NodeId, antlr4_runtime::AntlrError>,
) -> Result<antlr4_runtime::ParsedFile, antlr4_runtime::AntlrError>
{{
parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry)
}}
/// Parses UTF-8 text and returns a typed tree whose required generated child
/// accessors are infallible.
pub fn parse_validated<L: TokenSource>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<antlr4_runtime::NodeId, antlr4_runtime::AntlrError>,
) -> Result<{validated_tree_name}, {validation_error_name}>
{{
parse_stream_validated(
antlr4_runtime::InputStream::new(input.as_ref()),
lexer,
entry,
)
}}
/// Parses UTF-8 text like [`parse`] while returning the parser after the entry
/// rule has run.
///
/// This keeps the compact generated setup path available for callers that also
/// need `Parser::number_of_syntax_errors()` or `{type_name}::into_token_stream()`.
pub fn parse_with_parser<L: TokenSource, R>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<R, antlr4_runtime::AntlrError>,
) -> Result<{output_type_name}<R, L>, antlr4_runtime::AntlrError>
{{
parse_stream_with_parser(
antlr4_runtime::InputStream::new(input.as_ref()),
lexer,
entry,
)
}}
/// Parses a caller-provided character stream by constructing the lexer, token
/// stream, parser, and caller-selected entry rule in one call.
///
/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including
/// a named [`antlr4_runtime::InputStream`] or a byte-oriented
/// [`antlr4_runtime::ByteStream`].
pub fn parse_stream<I: antlr4_runtime::CharStream, L: TokenSource>(
input: I,
lexer: impl FnOnce(I) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<antlr4_runtime::NodeId, antlr4_runtime::AntlrError>,
) -> Result<antlr4_runtime::ParsedFile, antlr4_runtime::AntlrError>
{{
let {output_type_name} {{ result, parser }} =
parse_stream_with_parser(input, lexer, entry)?;
Ok(parser.into_parsed_file(result))
}}
/// Parses a caller-provided character stream and validates the completed tree.
pub fn parse_stream_validated<I: antlr4_runtime::CharStream, L: TokenSource>(
input: I,
lexer: impl FnOnce(I) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<antlr4_runtime::NodeId, antlr4_runtime::AntlrError>,
) -> Result<{validated_tree_name}, {validation_error_name}>
{{
let output = parse_stream_with_parser(input, lexer, entry)
.map_err({validation_error_name}::Recognition)?;
output.validate()
}}
/// Parses a caller-provided character stream like [`parse_stream`] while
/// returning the parser after the entry rule has run.
pub fn parse_stream_with_parser<I: antlr4_runtime::CharStream, L: TokenSource, R>(
input: I,
lexer: impl FnOnce(I) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<R, antlr4_runtime::AntlrError>,
) -> Result<{output_type_name}<R, L>, antlr4_runtime::AntlrError>
{{
let lexer = lexer(input);
let tokens = CommonTokenStream::new(lexer);
let mut parser = {type_name}::new(tokens);
let result = entry(&mut parser)?;
Ok({output_type_name} {{ result, parser }})
}}"#
)
}