use std::collections::{BTreeMap, BTreeSet, btree_map::Entry};
use std::env;
use std::fmt::Write as _;
use std::fs;
use std::io::{self, Write as _};
use std::ops::AddAssign;
use std::path::{Path, PathBuf};
use antlr4_runtime::atn::lexer_dfa::CompiledLexerDfa;
use antlr4_runtime::atn::parser_atn::{
ParserAtn, ParserAtnState, ParserTokenSetKind, ParserTransition, ParserTransitionData,
};
use antlr4_runtime::atn::{AtnStateKind, LexerAction, LexerAtn, LexerTransition};
use antlr4_runtime::token::TOKEN_EOF;
use grammar::atn::{CompiledLexer, CompiledParser, FinalizedAtnGraph, FinalizedTransitionKind};
use grammar::frontend::SourceSpan;
use grammar::loader::LoadOptions;
use grammar::model::{
Alternative, AlternativeId, AttributeSymbol, Block, Element, ElementKind, LabelKind,
LeftRecursiveAlternativeKind, ModelNodeId, Quantifier, Rule, SemanticGrammar, SetElement,
Terminal, Vocabulary,
};
use grammar::provenance::{Origin, ProvenanceIndex};
use grammar::source::SourceSet;
use petgraph::graph::DiGraph;
use petgraph::visit::{Dfs, Reversed};
#[path = "../bin_support/embedded.rs"]
mod embedded;
#[allow(dead_code)]
#[path = "../bin_support/grammar/mod.rs"]
mod grammar;
#[path = "../bin_support/rust_names.rs"]
mod rust_names;
#[path = "../bin_support/templates.rs"]
mod templates;
#[cfg(test)]
use rust_names::is_rust_keyword;
use rust_names::{
module_name, rust_function_name, rust_string, rust_type_name, sanitize_identifier,
split_identifier_words,
};
use templates::{
matching_template_close, parse_template_string, split_template_arguments,
template_sequence_bodies,
};
const GENERATED_MODULE_HEADER: &str = concat!(
"// @generated by ",
env!("CARGO_PKG_NAME"),
" v",
env!("CARGO_PKG_VERSION"),
" - do not edit\n",
"// project: ",
env!("CARGO_PKG_REPOSITORY"),
"\n",
"#[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]\n",
"#[rustfmt::skip]\n",
"mod __antlr4_rust_generated {\n",
"\n",
);
const GENERATED_MODULE_FOOTER: &str = "\
}
#[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]
pub use self::__antlr4_rust_generated::*;
";
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = match Args::parse()? {
CliCommand::Generate(args) => args,
CliCommand::Help => {
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", usage())?;
return Ok(());
}
CliCommand::Version => {
let mut stdout = io::stdout().lock();
writeln!(stdout, "antlr4-rust-gen {}", env!("CARGO_PKG_VERSION"))?;
return Ok(());
}
};
let compilation = grammar::compiler::compile(LoadOptions {
roots: args.roots.clone(),
library_directories: args.library_directories.clone(),
})
.map_err(|error| render_compilation_error(&error, &args.roots))?;
emit_compilation_warnings(&compilation)?;
let mut grammar_options = Vec::new();
let mut manifest_grammars: Vec<(&'static str, String, Vec<SemanticsEntry>)> = Vec::new();
let mut rendered_modules = BTreeMap::<PathBuf, String>::new();
let mut emitted_lexers = BTreeSet::new();
let mut emitted_parsers = BTreeSet::new();
for root in &compilation.roots {
if let Some(grammar) = root.lexer
&& emitted_lexers.insert(grammar)
{
let compiled = compilation
.lexer(grammar)
.expect("compiled root lexer artifact exists");
let data = CodegenData::from_lexer(compiled, &compilation.sources);
grammar_options.extend(collect_structural_grammar_options(
&data,
&args.option_hooks,
)?);
let entries = collect_lexer_semantics(
&data,
args.embedded_actions,
args.allow_unsupported_lexer_actions,
args.sem_unknown,
&args.sem_patterns,
)?;
enforce_sem_unknown(args.sem_unknown, &entries)?;
enforce_require_full_semantics(args.require_full_semantics, &entries)?;
let grammar_name = compiled.semantic.recognizer.name.clone();
let module = render_lexer(
&grammar_name,
&data,
args.allow_unsupported_lexer_actions,
args.sem_unknown,
&args.sem_patterns,
args.embedded_actions,
)?;
insert_rendered_module(&mut rendered_modules, &grammar_name, module)?;
manifest_grammars.push(("lexer", grammar_name, entries));
}
if let Some(grammar) = root.parser
&& emitted_parsers.insert(grammar)
{
let compiled = compilation
.parser(grammar)
.expect("compiled root parser artifact exists");
let data = CodegenData::from_parser(compiled, &compilation.sources);
grammar_options.extend(collect_structural_grammar_options(
&data,
&args.option_hooks,
)?);
let entries = collect_parser_semantics_for_mode(
&data,
args.embedded_actions,
args.sem_unknown,
&args.sem_patterns,
)?;
enforce_sem_unknown(args.sem_unknown, &entries)?;
enforce_require_full_semantics(args.require_full_semantics, &entries)?;
let grammar_name = compiled.semantic.recognizer.name.clone();
let module = render_parser_with_options(
&grammar_name,
&data,
ParserRenderOptions {
require_generated_parser: args.require_generated_parser,
embedded: args.embedded_actions,
generate_listener: args.generate_listener,
generate_visitor: args.generate_visitor,
sem_unknown: args.sem_unknown,
patterns: Some(&args.sem_patterns),
},
)?;
insert_rendered_module(&mut rendered_modules, &grammar_name, module)?;
manifest_grammars.push(("parser", grammar_name, entries));
}
}
deduplicate_grammar_options(&mut grammar_options);
emit_grammar_option_warnings(&grammar_options)?;
enforce_require_full_options(args.require_full_semantics, &grammar_options)?;
let manifest =
render_semantics_manifest(args.sem_unknown, &grammar_options, &manifest_grammars);
fs::create_dir_all(&args.out_dir)?;
for (path, module) in rendered_modules {
fs::write(args.out_dir.join(path), module)?;
}
fs::write(args.out_dir.join("semantics.json"), manifest)?;
Ok(())
}
fn insert_rendered_module(
modules: &mut BTreeMap<PathBuf, String>,
grammar_name: &str,
module: String,
) -> io::Result<()> {
let path = PathBuf::from(format!("{}.rs", module_name(grammar_name)));
match modules.entry(path.clone()) {
Entry::Vacant(entry) => {
entry.insert(module);
Ok(())
}
Entry::Occupied(_) => Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"generated module collision for grammar {grammar_name}: {}",
path.display()
),
)),
}
}
fn deduplicate_grammar_options(options: &mut Vec<GrammarOptionEntry>) {
options.sort_by(|left, right| {
(&left.key, &left.value, left.line, left.column).cmp(&(
&right.key,
&right.value,
right.line,
right.column,
))
});
options.dedup_by(|left, right| {
left.key == right.key
&& left.value == right.value
&& left.line == right.line
&& left.column == right.column
});
}
fn render_compilation_error(
error: &grammar::diagnostic::CompilationError,
roots: &[PathBuf],
) -> io::Error {
let fallback = roots
.first()
.map_or_else(|| Path::new("<grammar>"), PathBuf::as_path);
let mut message = String::new();
for (index, diagnostic) in error.diagnostics().iter().enumerate() {
let severity = match diagnostic.severity {
grammar::diagnostic::Severity::Warning => "warning",
grammar::diagnostic::Severity::Error => "error",
};
let location = error.location(index);
let path = location.map_or_else(
|| fallback.display().to_string(),
|location| location.path.display().to_string(),
);
let position = location
.and_then(|location| location.position)
.map_or_else(String::new, |(line, column)| format!(":{line}:{column}"));
let _ = writeln!(
message,
"{severity}[{}]: {path}{position}: {}",
diagnostic.code, diagnostic.message
);
}
io::Error::new(io::ErrorKind::InvalidInput, message)
}
fn emit_compilation_warnings(compilation: &grammar::compiler::Compilation) -> io::Result<()> {
let mut stderr = io::stderr().lock();
for diagnostic in compilation
.diagnostics
.iter()
.filter(|diagnostic| diagnostic.severity == grammar::diagnostic::Severity::Warning)
{
let source = compilation.sources.get(diagnostic.primary.source);
let path = source.map_or_else(
|| "<grammar>".to_owned(),
|source| source.logical_path().display().to_string(),
);
let position = source
.and_then(|source| source.line_column(diagnostic.primary.bytes.start))
.map_or_else(String::new, |(line, column)| format!(":{line}:{column}"));
writeln!(
stderr,
"warning[{}]: {path}{position}: {}",
diagnostic.code, diagnostic.message
)?;
}
Ok(())
}
/// Disposition policy for semantic predicate/action coordinates that the
/// generator cannot translate into runtime metadata.
///
/// Mirrors the runtime's `UnknownSemanticPolicy`; the generator additionally
/// uses [`Self::Error`] to fail code generation before emitting a module whose
/// semantics would be unreliable.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum SemUnknownPolicy {
/// Unknown predicates pass unconditionally and unknown actions are
/// no-ops. Matches the historical metadata-only behavior; deprecated as a
/// default and slated to change to [`Self::Error`] in a future minor
/// release.
#[default]
AssumeTrue,
/// Unknown predicates fail unconditionally; unknown actions remain
/// no-ops.
AssumeFalse,
/// Unknown coordinates are intentionally delegated to runtime hooks.
Hook,
/// Fail code generation when any semantic coordinate has no Rust
/// implementation.
Error,
}
impl SemUnknownPolicy {
fn parse_flag(value: &str) -> Result<Self, String> {
match value {
"error" => Ok(Self::Error),
"hook" => Ok(Self::Hook),
"assume-true" => Ok(Self::AssumeTrue),
"assume-false" => Ok(Self::AssumeFalse),
other => Err(format!(
"--sem-unknown accepts error, hook, assume-true, or assume-false; got {other}\n\n{}",
usage()
)),
}
}
const fn manifest_name(self) -> &'static str {
match self {
Self::AssumeTrue => "assume-true",
Self::AssumeFalse => "assume-false",
Self::Hook => "hook",
Self::Error => "error",
}
}
/// Manifest disposition recorded for a predicate coordinate that has no
/// translated template. [`Self::Error`] aborts generation before a
/// manifest is written, so its mapping is only ever read by the
/// fail-loud report.
const fn unknown_predicate_disposition(self) -> SemanticsDisposition {
match self {
Self::AssumeTrue | Self::Error => SemanticsDisposition::AssumeTrue,
Self::AssumeFalse => SemanticsDisposition::AssumeFalse,
Self::Hook => SemanticsDisposition::Hooked,
}
}
const fn unknown_action_disposition(self) -> SemanticsDisposition {
match self {
Self::Hook => SemanticsDisposition::Hooked,
Self::AssumeTrue | Self::AssumeFalse | Self::Error => SemanticsDisposition::Ignored,
}
}
}
/// Coordinate kinds tracked by the `semantics.json` manifest.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SemanticsKind {
LexerAction,
LexerPredicate,
ParserPredicate,
ParserAction,
}
impl SemanticsKind {
const fn manifest_name(self) -> &'static str {
match self {
Self::LexerAction => "lexer-action",
Self::LexerPredicate => "lexer-predicate",
Self::ParserPredicate => "parser-predicate",
Self::ParserAction => "parser-action",
}
}
const fn error_label(self) -> &'static str {
match self {
Self::LexerAction => "unsupported grammar action",
Self::LexerPredicate | Self::ParserPredicate => "unsupported semantic predicate",
Self::ParserAction => "unsupported grammar action",
}
}
}
/// How generation disposed of one semantic coordinate.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SemanticsDisposition {
/// A supported template translated the coordinate into runtime metadata
/// or generated Rust code.
Translated,
/// No implementation exists; recognition treats the predicate as passing.
AssumeTrue,
/// No implementation exists; recognition treats the predicate as failing.
AssumeFalse,
/// No generated implementation exists; runtime hooks own the coordinate.
Hooked,
/// The coordinate is intentionally rejected by policy.
Error,
/// No implementation exists; the action is a no-op at recognition time.
Ignored,
/// An action ANTLR synthesized (e.g. during left-recursion elimination),
/// not written by the grammar author. It is a no-op at recognition time
/// like [`Self::Ignored`], but carries no author intent, so it is exempt
/// from the `--sem-unknown=error` gate.
Synthetic,
}
impl SemanticsDisposition {
const fn manifest_name(self) -> &'static str {
match self {
Self::Translated => "translated",
Self::AssumeTrue => "assume-true",
Self::AssumeFalse => "assume-false",
Self::Hooked => "hooked",
Self::Error => "error",
Self::Ignored => "ignored",
Self::Synthetic => "synthetic",
}
}
}
/// How the Rust backend treats one top-level ANTLR grammar option.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GrammarOptionDisposition {
/// The direct compiler represented the option in the compiled artifacts.
Metadata,
/// The option affects the upstream tool invocation, not Rust runtime
/// behavior.
ToolHandled,
/// Caller-owned Rust hooks explicitly provide the target behavior.
Hooked,
/// The option is legal ANTLR syntax but its target behavior is not
/// automatically implemented by the Rust backend.
Unsupported,
}
impl GrammarOptionDisposition {
const fn manifest_name(self) -> &'static str {
match self {
Self::Metadata => "metadata",
Self::ToolHandled => "tool-handled",
Self::Hooked => "hooked",
Self::Unsupported => "unsupported",
}
}
}
/// One source-level grammar option inventoried from the compilation.
#[derive(Clone, Debug, Eq, PartialEq)]
struct GrammarOptionEntry {
key: String,
value: String,
line: usize,
column: usize,
disposition: GrammarOptionDisposition,
}
impl GrammarOptionEntry {
fn assignment(&self) -> String {
format!("{}={}", self.key, self.value)
}
fn describe_unsupported(&self) -> String {
format!(
"unsupported grammar option: {} at {}:{} is accepted by ANTLR but is not \
automatically implemented by antlr4-rust-gen; target-specific behavior may be \
missing (see #98)",
self.assignment(),
self.line,
self.column
)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum CoordinateDispose {
Hook,
AssumeTrue,
AssumeFalse,
Error,
}
impl CoordinateDispose {
fn parse(value: &str) -> io::Result<Self> {
match value {
"hook" => Ok(Self::Hook),
"assume-true" => Ok(Self::AssumeTrue),
"assume-false" => Ok(Self::AssumeFalse),
"error" => Ok(Self::Error),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown coordinate dispose {other}"),
)),
}
}
const fn disposition(self) -> SemanticsDisposition {
match self {
Self::Hook => SemanticsDisposition::Hooked,
Self::AssumeTrue => SemanticsDisposition::AssumeTrue,
Self::AssumeFalse => SemanticsDisposition::AssumeFalse,
Self::Error => SemanticsDisposition::Error,
}
}
const fn predicate_template(self) -> Option<PredicateTemplate> {
match self {
Self::Hook => Some(PredicateTemplate::Hook),
Self::AssumeTrue => Some(PredicateTemplate::True),
Self::AssumeFalse => Some(PredicateTemplate::False),
Self::Error => None,
}
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
struct SemPatternFile {
patterns: Vec<SemPatternRule>,
helpers: Vec<SemHelperRule>,
coordinates: Vec<SemCoordinateOverride>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemPatternRule {
id: String,
match_body: String,
lower: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemHelperRule {
/// Explicit helper kind. A missing kind preserves the pre-v1 wildcard
/// behavior for lexer and parser predicates.
kind: Option<SemanticsKind>,
name: String,
arguments: Vec<SemanticLiteralKind>,
lower: String,
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
enum SemanticLiteralKind {
String,
Bool,
Integer,
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum SemanticLiteral {
String(String),
Bool(bool),
Integer(i64),
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemanticHelperCall {
name: String,
arguments: Vec<SemanticLiteral>,
negated: bool,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemCoordinateOverride {
kind: SemanticsKind,
rule: Option<String>,
index: Option<usize>,
atn_state: Option<usize>,
dispose: CoordinateDispose,
}
impl SemPatternFile {
fn predicate_template(
&self,
kind: SemanticsKind,
body: &str,
) -> io::Result<Option<PredicateTemplate>> {
let body = body.trim();
let mut matches = Vec::new();
matches.extend(
self.patterns
.iter()
.filter(|pattern| pattern.match_body.trim() == body)
.map(|pattern| (pattern.id.as_str(), pattern.lower.as_str())),
);
matches.extend(
self.helpers
.iter()
.filter(|helper| semantic_helper_kind_matches(helper, kind))
.filter_map(|helper| {
parse_semantic_helper_call(body, kind)
.filter(|call| helper_call_matches(call, helper))
.map(|_| helper)
})
.map(|helper| (helper.name.as_str(), helper.lower.as_str())),
);
if matches.len() > 1 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"ambiguous semantic patterns for {body:?}: {}",
matches
.iter()
.map(|(id, _)| *id)
.collect::<Vec<_>>()
.join(", ")
),
));
}
matches
.first()
.map(|(_, lower)| parse_pattern_lower(lower))
.transpose()
}
fn hook_helper_call(
&self,
kind: SemanticsKind,
body: &str,
) -> io::Result<Option<SemanticHelperCall>> {
let Some(call) = parse_semantic_helper_call(body, kind) else {
return Ok(None);
};
let matches = self
.helpers
.iter()
.filter(|helper| {
semantic_helper_kind_matches(helper, kind) && helper.lower.trim() == "hook"
})
.filter(|helper| helper_call_matches(&call, helper))
.collect::<Vec<_>>();
if matches.len() > 1 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"ambiguous semantic helper patterns for {body:?}: {}",
matches
.iter()
.map(|helper| helper.name.as_str())
.collect::<Vec<_>>()
.join(", ")
),
));
}
Ok(matches.first().map(|_| call))
}
fn coordinate_disposition(
&self,
kind: SemanticsKind,
rule: Option<&str>,
index: Option<usize>,
atn_state: Option<usize>,
) -> Option<SemanticsDisposition> {
self.coordinate_override(kind, rule, index, atn_state)
.map(|override_| override_.dispose.disposition())
}
#[allow(clippy::option_option)]
fn coordinate_predicate_template(
&self,
kind: SemanticsKind,
rule: Option<&str>,
index: Option<usize>,
) -> Option<Option<PredicateTemplate>> {
self.coordinate_override(kind, rule, index, None)
.map(|override_| override_.dispose.predicate_template())
}
fn coordinate_override(
&self,
kind: SemanticsKind,
rule: Option<&str>,
index: Option<usize>,
atn_state: Option<usize>,
) -> Option<&SemCoordinateOverride> {
self.coordinates.iter().find(|override_| {
override_.kind == kind
&& override_
.rule
.as_deref()
.is_none_or(|expected| rule == Some(expected))
&& override_
.index
.is_none_or(|expected| index == Some(expected))
&& override_
.atn_state
.is_none_or(|expected| atn_state == Some(expected))
})
}
}
fn semantic_helper_kind_matches(helper: &SemHelperRule, kind: SemanticsKind) -> bool {
helper.kind == Some(kind)
|| (helper.kind.is_none()
&& matches!(
kind,
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate
))
}
fn helper_call_matches(call: &SemanticHelperCall, helper: &SemHelperRule) -> bool {
call.name == helper.name
&& call.arguments.len() == helper.arguments.len()
&& call
.arguments
.iter()
.zip(&helper.arguments)
.all(|(literal, kind)| {
matches!(
(literal, kind),
(SemanticLiteral::String(_), SemanticLiteralKind::String)
| (SemanticLiteral::Bool(_), SemanticLiteralKind::Bool)
| (SemanticLiteral::Integer(_), SemanticLiteralKind::Integer)
)
})
}
fn parse_semantic_helper_call(body: &str, kind: SemanticsKind) -> Option<SemanticHelperCall> {
let mut body = body.trim();
if matches!(
kind,
SemanticsKind::LexerAction | SemanticsKind::ParserAction
) {
body = body.strip_suffix(';').unwrap_or(body).trim_end();
}
let negated = matches!(
kind,
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate
) && body.starts_with('!');
if negated {
body = body[1..].trim_start();
}
body = body
.strip_prefix("this.")
.or_else(|| body.strip_prefix("self."))
.unwrap_or(body);
let open = body.find('(')?;
let name = body[..open].trim();
if !is_semantic_helper_identifier(name) || !body.ends_with(')') {
return None;
}
let arguments = parse_semantic_literals(&body[open + 1..body.len() - 1])?;
Some(SemanticHelperCall {
name: name.to_owned(),
arguments,
negated,
})
}
fn is_semantic_helper_identifier(value: &str) -> bool {
let mut chars = value.chars();
chars.next().is_some_and(|first| {
(first == '_' || first.is_ascii_alphabetic())
&& chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
})
}
fn parse_semantic_literals(body: &str) -> Option<Vec<SemanticLiteral>> {
let mut body = body.trim();
if body.is_empty() {
return Some(Vec::new());
}
let mut literals = Vec::new();
while !body.is_empty() {
if body.starts_with('"') || body.starts_with('\'') {
let quote = *body.as_bytes().first()?;
let mut escaped = false;
let mut end = None;
for (index, byte) in body.as_bytes().iter().copied().enumerate().skip(1) {
if escaped {
escaped = false;
} else if byte == b'\\' {
escaped = true;
} else if byte == quote {
end = Some(index);
break;
}
}
let end = end?;
let raw = &body[1..end];
let value = unescape_semantic_string(raw)?;
literals.push(SemanticLiteral::String(value));
body = body[end + 1..].trim_start();
} else {
let end = body.find(',').unwrap_or(body.len());
let raw = body[..end].trim();
let literal = match raw {
"true" => SemanticLiteral::Bool(true),
"false" => SemanticLiteral::Bool(false),
_ => SemanticLiteral::Integer(raw.parse().ok()?),
};
literals.push(literal);
body = body[end..].trim_start();
}
if body.is_empty() {
break;
}
body = body.strip_prefix(',')?.trim_start();
if body.is_empty() {
return None;
}
}
Some(literals)
}
fn unescape_semantic_string(value: &str) -> Option<String> {
let mut out = String::with_capacity(value.len());
let mut chars = value.chars();
while let Some(ch) = chars.next() {
if ch != '\\' {
out.push(ch);
continue;
}
match chars.next()? {
'"' => out.push('"'),
'\'' => out.push('\''),
'\\' => out.push('\\'),
'n' => out.push('\n'),
'r' => out.push('\r'),
't' => out.push('\t'),
other => {
out.push('\\');
out.push(other);
}
}
}
Some(out)
}
fn parse_pattern_lower(lower: &str) -> io::Result<PredicateTemplate> {
let lower = lower.trim();
match lower {
"true" | "bool(true)" => return Ok(PredicateTemplate::True),
"false" | "bool(false)" => return Ok(PredicateTemplate::False),
"hook" => return Ok(PredicateTemplate::Hook),
_ => {}
}
parse_pattern_lt_text(lower)
.or_else(|| parse_pattern_la_not_equals(lower))
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported semantic pattern lower expression {lower:?}"),
)
})
}
fn parse_pattern_lt_text(lower: &str) -> Option<PredicateTemplate> {
let body = lower.strip_prefix("cmp(eq, token_text(")?;
let (offset, rest) = body.split_once("), str(\"")?;
let text = rest.strip_suffix("\"))")?;
Some(PredicateTemplate::LookaheadTextEquals {
offset: offset.trim().parse().ok()?,
text: text.to_owned(),
})
}
fn parse_pattern_la_not_equals(lower: &str) -> Option<PredicateTemplate> {
let body = lower.strip_prefix("cmp(ne, la(")?;
let (offset, rest) = body.split_once("), token(")?;
let token_name = rest.strip_suffix("))")?;
Some(PredicateTemplate::LookaheadNotEquals {
offset: offset.trim().parse().ok()?,
token_name: token_name.trim().to_owned(),
})
}
/// One semantic predicate/action coordinate inventoried for the manifest.
///
/// Optional fields reflect the coordinate kind or the absence of authored
/// source for a synthetic action. Authored bodies and spans come from the same
/// structural element that owns the finalized ATN transition.
#[derive(Clone, Debug)]
struct SemanticsEntry {
kind: SemanticsKind,
rule_index: Option<usize>,
rule_name: Option<String>,
index: Option<usize>,
atn_state: Option<usize>,
line: Option<usize>,
column: Option<usize>,
body: Option<String>,
disposition: SemanticsDisposition,
template: Option<String>,
}
impl SemanticsEntry {
/// Renders the fail-loud error line for this coordinate, matching the
/// shape documented in the compatibility-boundary docs.
fn describe_unsupported(&self) -> String {
let mut message = String::from(self.kind.error_label());
message.push(':');
match (&self.rule_name, self.rule_index) {
(Some(name), Some(rule_index)) => {
let _ = write!(message, " rule={name}({rule_index})");
}
(None, Some(rule_index)) => {
let _ = write!(message, " rule_index={rule_index}");
}
_ => {}
}
if let Some(index) = self.index {
let label = match self.kind {
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate => "pred_index",
SemanticsKind::LexerAction | SemanticsKind::ParserAction => "action_index",
};
let _ = write!(message, " {label}={index}");
}
if let Some(atn_state) = self.atn_state {
let _ = write!(message, " atn_state={atn_state}");
}
if let (Some(line), Some(column)) = (self.line, self.column) {
let _ = write!(message, " at {line}:{column}");
}
if let Some(body) = &self.body {
let _ = write!(message, ": {{{body}}}");
}
message
}
}
/// Fails generation when coordinates must be rejected at codegen: either the
/// global `--sem-unknown=error` policy is active (every unimplemented
/// coordinate), or a per-coordinate `dispose = "error"` override rejects a
/// specific coordinate regardless of the global policy.
///
/// A per-coordinate error override otherwise lowers to no `SemIR` entry and
/// does not escalate the runtime policy, so without this it would silently fall
/// back to the global default (e.g. `AssumeTrue`) instead of failing.
fn enforce_sem_unknown(policy: SemUnknownPolicy, entries: &[SemanticsEntry]) -> io::Result<()> {
let unsupported = entries
.iter()
.filter(|entry| {
// A per-coordinate `dispose = "error"` override is always fatal.
if entry.disposition == SemanticsDisposition::Error {
return true;
}
// Under the global Error policy, any unimplemented coordinate is
// fatal — except a `Synthetic` action ANTLR inserted itself (no
// author intent to implement).
policy == SemUnknownPolicy::Error
&& !matches!(
entry.disposition,
SemanticsDisposition::Translated
| SemanticsDisposition::Hooked
| SemanticsDisposition::Synthetic
)
})
.collect::<Vec<_>>();
if unsupported.is_empty() {
return Ok(());
}
let mut message = String::new();
for entry in &unsupported {
message.push_str(&entry.describe_unsupported());
message.push('\n');
}
let _ = write!(
message,
"--sem-unknown=error: {} semantic coordinate(s) have no Rust implementation and are \
configured to fail; add a semantic pattern, adjust a coordinate's `dispose`, or accept \
a documented fallback with \
--sem-unknown=assume-true / assume-false",
unsupported.len()
);
Err(io::Error::new(io::ErrorKind::InvalidData, message))
}
fn enforce_require_full_semantics(require: bool, entries: &[SemanticsEntry]) -> io::Result<()> {
if !require {
return Ok(());
}
let fallback = entries
.iter()
.filter(|entry| {
// A `Synthetic` action is an ANTLR internal, not a missing author
// semantic, so it does not count as an unimplemented fallback.
!matches!(
entry.disposition,
SemanticsDisposition::Translated
| SemanticsDisposition::Hooked
| SemanticsDisposition::Synthetic
)
})
.collect::<Vec<_>>();
if fallback.is_empty() {
return Ok(());
}
let mut message = String::new();
for entry in &fallback {
message.push_str(&entry.describe_unsupported());
message.push('\n');
}
let _ = write!(
message,
"--require-full-semantics: {} semantic coordinate(s) use policy fallback dispositions",
fallback.len()
);
Err(io::Error::new(io::ErrorKind::InvalidData, message))
}
fn emit_grammar_option_warnings(entries: &[GrammarOptionEntry]) -> io::Result<()> {
let mut stderr = io::stderr().lock();
for entry in entries
.iter()
.filter(|entry| entry.disposition == GrammarOptionDisposition::Unsupported)
{
writeln!(stderr, "warning: {}", entry.describe_unsupported())?;
}
Ok(())
}
fn enforce_require_full_options(require: bool, entries: &[GrammarOptionEntry]) -> io::Result<()> {
if !require {
return Ok(());
}
let unsupported = entries
.iter()
.filter(|entry| entry.disposition == GrammarOptionDisposition::Unsupported)
.collect::<Vec<_>>();
if unsupported.is_empty() {
return Ok(());
}
let mut message = String::new();
for entry in &unsupported {
message.push_str(&entry.describe_unsupported());
message.push('\n');
}
let _ = write!(
message,
"--require-full-semantics: {} grammar option(s) require caller-owned target behavior; \
acknowledge implemented behavior with --option-hook KEY=VALUE",
unsupported.len()
);
Err(io::Error::new(io::ErrorKind::InvalidData, message))
}
fn normalize_option_hook(value: &str) -> Result<String, String> {
let Some((key, option_value)) = value.split_once('=') else {
return Err(format!(
"--option-hook requires KEY=VALUE; got {value:?}\n\n{}",
usage()
));
};
let key = key.trim();
let option_value = option_value.trim();
let mut key_chars = key.chars();
if !key_chars
.next()
.is_some_and(|ch| ch == '_' || ch.is_ascii_alphabetic())
|| !key_chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
|| option_value.is_empty()
{
return Err(format!(
"--option-hook requires KEY=VALUE; got {value:?}\n\n{}",
usage()
));
}
Ok(format!("{key}={option_value}"))
}
/// Classifies every structurally bound custom-action and predicate coordinate
/// so manifest dispositions match what the generated module will do.
/// Manifest disposition for a predicate coordinate given its covering
/// template: hook-routed coordinates report `hooked` (the user trait owns
/// them), any other template is a real translation, and uncovered
/// coordinates fall back to the unknown policy.
const fn predicate_template_disposition(
template: Option<&PredicateTemplate>,
policy: SemUnknownPolicy,
) -> SemanticsDisposition {
match template {
// Unwrap a `<fail=...>` wrapper: disposition follows the inner template
// (a wrapped hook is still `Hooked`, not `Translated`).
Some(PredicateTemplate::WithFailMessage { inner, .. }) => {
predicate_template_disposition(Some(inner), policy)
}
Some(PredicateTemplate::Hook) => SemanticsDisposition::Hooked,
Some(_) => SemanticsDisposition::Translated,
None => policy.unknown_predicate_disposition(),
}
}
fn collect_lexer_semantics(
data: &CodegenData<'_>,
embedded: bool,
allow_unsupported_lexer_actions: bool,
policy: SemUnknownPolicy,
patterns: &SemPatternFile,
) -> io::Result<Vec<SemanticsEntry>> {
let actions = if embedded {
Vec::new()
} else {
let actions = structural_lexer_action_templates(data, patterns)?;
reject_unsupported_lexer_action_templates(
&actions
.iter()
.map(|(_, template)| template.clone())
.collect::<Vec<_>>(),
allow_unsupported_lexer_actions,
)?;
actions
};
let predicates = if embedded {
Vec::new()
} else {
structural_predicate_templates(data, SemanticsKind::LexerPredicate, patterns)?
};
let mut entries = Vec::new();
for action in structural_actions(data)? {
let coordinate = (
i32::try_from(action.rule_index).expect("rule index exceeds i32"),
i32::try_from(action.action_index).expect("action index exceeds i32"),
);
let template = actions
.iter()
.find(|(covered, _)| *covered == coordinate)
.map(|(_, template)| template);
let (line, column) = structural_line_column(data, &action.span);
entries.push(SemanticsEntry {
kind: SemanticsKind::LexerAction,
rule_index: Some(action.rule_index),
rule_name: data.rule_names.get(action.rule_index).cloned(),
index: Some(action.action_index),
atn_state: None,
line: Some(line),
column: Some(column),
body: Some(one_line_action_body(&action.body)),
disposition: if embedded {
SemanticsDisposition::Translated
} else {
patterns
.coordinate_disposition(
SemanticsKind::LexerAction,
data.rule_names.get(action.rule_index).map(String::as_str),
Some(action.action_index),
None,
)
.unwrap_or_else(|| match template {
Some(ActionTemplate::Hook(_)) => SemanticsDisposition::Hooked,
Some(ActionTemplate::LexerPopMode) => SemanticsDisposition::Translated,
Some(ActionTemplate::UnsupportedLexerAction { .. }) | None => {
policy.unknown_action_disposition()
}
})
},
template: if embedded {
Some("Embedded".to_owned())
} else {
matches!(template, Some(ActionTemplate::LexerPopMode))
.then(|| format!("{:?}", template.expect("matched template")))
},
});
}
for predicate in structural_predicates(data)? {
let coordinate = (predicate.rule_index, predicate.predicate_index);
let template = predicates
.iter()
.find(|(covered, _)| *covered == coordinate)
.map(|(_, template)| template);
let (line, column) = structural_line_column(data, &predicate.span);
entries.push(SemanticsEntry {
kind: SemanticsKind::LexerPredicate,
rule_index: Some(predicate.rule_index),
rule_name: data.rule_names.get(predicate.rule_index).cloned(),
index: Some(predicate.predicate_index),
atn_state: None,
line: Some(line),
column: Some(column),
body: Some(one_line_action_body(&predicate.body)),
disposition: if embedded {
SemanticsDisposition::Translated
} else {
patterns
.coordinate_disposition(
SemanticsKind::LexerPredicate,
data.rule_names
.get(predicate.rule_index)
.map(String::as_str),
Some(predicate.predicate_index),
None,
)
.unwrap_or_else(|| predicate_template_disposition(template, policy))
},
template: if embedded {
Some("Embedded".to_owned())
} else {
template.map(|template| format!("{template:?}"))
},
});
}
Ok(entries)
}
#[cfg(test)]
fn collect_parser_semantics(
data: &CodegenData<'_>,
policy: SemUnknownPolicy,
patterns: &SemPatternFile,
) -> io::Result<Vec<SemanticsEntry>> {
collect_parser_semantics_for_mode(data, false, policy, patterns)
}
fn collect_parser_semantics_for_mode(
data: &CodegenData<'_>,
embedded: bool,
policy: SemUnknownPolicy,
patterns: &SemPatternFile,
) -> io::Result<Vec<SemanticsEntry>> {
let portable_locals = (!embedded)
.then(|| build_structural_portable_local_data(data, patterns))
.transpose()?
.unwrap_or_default();
let predicates = if embedded {
Vec::new()
} else {
structural_predicate_templates(data, SemanticsKind::ParserPredicate, patterns)?
};
let mut entries = Vec::new();
for predicate in structural_predicates(data)? {
let coordinate = (predicate.rule_index, predicate.predicate_index);
let template = predicates
.iter()
.find(|(covered, _)| *covered == coordinate)
.map(|(_, template)| template);
let (line, column) = structural_line_column(data, &predicate.span);
entries.push(SemanticsEntry {
kind: SemanticsKind::ParserPredicate,
rule_index: Some(predicate.rule_index),
rule_name: data.rule_names.get(predicate.rule_index).cloned(),
index: Some(predicate.predicate_index),
atn_state: None,
line: Some(line),
column: Some(column),
body: Some(one_line_action_body(&predicate.body)),
disposition: if embedded {
SemanticsDisposition::Translated
} else {
patterns
.coordinate_disposition(
SemanticsKind::ParserPredicate,
data.rule_names
.get(predicate.rule_index)
.map(String::as_str),
Some(predicate.predicate_index),
None,
)
.unwrap_or_else(|| {
if portable_locals.predicates.contains_key(&coordinate) {
SemanticsDisposition::Translated
} else {
predicate_template_disposition(template, policy)
}
})
},
template: if embedded {
Some("Embedded".to_owned())
} else {
portable_locals
.predicates
.contains_key(&coordinate)
.then(|| "PortableBooleanLocal".to_owned())
.or_else(|| template.map(|template| format!("{template:?}")))
},
});
}
for action in structural_actions(data)? {
let (line, column) = structural_line_column(data, &action.span);
entries.push(SemanticsEntry {
kind: SemanticsKind::ParserAction,
rule_index: Some(action.rule_index),
rule_name: data.rule_names.get(action.rule_index).cloned(),
index: None,
atn_state: Some(action.state),
line: action.authored.then_some(line),
column: action.authored.then_some(column),
body: action.authored.then(|| one_line_action_body(&action.body)),
disposition: if embedded && action.authored {
SemanticsDisposition::Translated
} else if embedded {
SemanticsDisposition::Synthetic
} else {
patterns
.coordinate_disposition(
SemanticsKind::ParserAction,
data.rule_names.get(action.rule_index).map(String::as_str),
None,
Some(action.state),
)
.unwrap_or_else(|| {
if portable_locals.inline_actions.contains_key(&action.state) {
SemanticsDisposition::Translated
} else if !action.authored || action.body.trim().is_empty() {
SemanticsDisposition::Synthetic
} else {
policy.unknown_action_disposition()
}
})
},
template: if embedded && action.authored {
Some("Embedded".to_owned())
} else if embedded {
None
} else {
portable_locals
.inline_actions
.contains_key(&action.state)
.then(|| "PortableBooleanLocal".to_owned())
},
});
}
entries.sort_by_key(|entry| {
(
entry.rule_index,
matches!(entry.kind, SemanticsKind::ParserAction),
entry.index,
entry.atn_state,
)
});
Ok(entries)
}
fn structural_lexer_action_templates(
data: &CodegenData<'_>,
patterns: &SemPatternFile,
) -> io::Result<Vec<((i32, i32), ActionTemplate)>> {
let mut templates = structural_actions(data)?
.into_iter()
.map(|action| {
let rule_name = data
.rule_names
.get(action.rule_index)
.map_or("<unknown>", String::as_str);
let template = match parse_lexer_action_block_template(&action.body) {
Some(template) => template,
None => patterns
.hook_helper_call(SemanticsKind::LexerAction, &action.body)?
.map_or_else(
|| ActionTemplate::UnsupportedLexerAction {
rule_name: rule_name.to_owned(),
body: one_line_action_body(&action.body),
},
ActionTemplate::Hook,
),
};
Ok((
(
i32::try_from(action.rule_index).expect("rule index exceeds i32"),
i32::try_from(action.action_index).expect("action index exceeds i32"),
),
template,
))
})
.collect::<io::Result<Vec<_>>>()?;
templates.sort_by_key(|(coordinate, _)| *coordinate);
templates.dedup_by_key(|(coordinate, _)| *coordinate);
Ok(templates)
}
fn structural_predicate_templates(
data: &CodegenData<'_>,
kind: SemanticsKind,
patterns: &SemPatternFile,
) -> io::Result<Vec<((usize, usize), PredicateTemplate)>> {
let mut templates = Vec::new();
for predicate in structural_predicates(data)? {
let coordinate = (predicate.rule_index, predicate.predicate_index);
let rule_name = data
.rule_names
.get(predicate.rule_index)
.map(String::as_str);
let (template, parsed_body) = match patterns.coordinate_predicate_template(
kind,
rule_name,
Some(predicate.predicate_index),
) {
Some(template) => (template, false),
None => (
parse_predicate_template_with_patterns_kind(&predicate.body, patterns, kind)?,
true,
),
};
if parsed_body && template.is_none() && is_unsupported_string_template_body(&predicate.body)
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported target predicate template <{}>", predicate.body),
));
}
if let Some(template) = template {
let template = match predicate.fail {
Some(message) => predicate_template_with_fail_message(template, message),
None => template,
};
templates.push((coordinate, template));
}
}
Ok(templates)
}
fn collect_structural_grammar_options(
data: &CodegenData<'_>,
option_hooks: &BTreeSet<String>,
) -> io::Result<Vec<GrammarOptionEntry>> {
let semantic = data.semantic.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"structural grammar model is unavailable",
)
})?;
Ok(semantic
.unit
.options
.iter()
.map(|option| {
let (line, column) = structural_line_column(data, &option.name.span);
let key = option.name.value.clone();
let value = option.value.value.clone();
let assignment = format!("{key}={value}");
let disposition = match key.as_str() {
"tokenVocab" | "caseInsensitive" => GrammarOptionDisposition::Metadata,
"language" => GrammarOptionDisposition::ToolHandled,
_ if option_hooks.contains(&assignment) => GrammarOptionDisposition::Hooked,
_ => GrammarOptionDisposition::Unsupported,
};
GrammarOptionEntry {
key,
value,
line,
column,
disposition,
}
})
.collect())
}
fn render_semantics_manifest(
policy: SemUnknownPolicy,
options: &[GrammarOptionEntry],
grammars: &[(&'static str, String, Vec<SemanticsEntry>)],
) -> String {
const DEPRECATION_NOTE: &str = "unknown coordinates currently default to assume-true; \
a future minor release changes the default to error";
let mut out = String::new();
out.push_str("{\n \"version\": 2,\n");
let _ = writeln!(
out,
" \"policy\": {},",
json_string(policy.manifest_name())
);
let _ = writeln!(out, " \"note\": {},", json_string(DEPRECATION_NOTE));
out.push_str(" \"options\": [");
for (position, option) in options.iter().enumerate() {
if position > 0 {
out.push(',');
}
out.push_str("\n ");
write_grammar_option_entry(&mut out, option);
}
if options.is_empty() {
out.push_str("],\n");
} else {
out.push_str("\n ],\n");
}
out.push_str(" \"grammars\": [");
for (grammar_position, (kind, name, entries)) in grammars.iter().enumerate() {
if grammar_position > 0 {
out.push(',');
}
out.push_str("\n {\n");
let _ = writeln!(out, " \"kind\": {},", json_string(kind));
let _ = writeln!(out, " \"name\": {},", json_string(name));
out.push_str(" \"coordinates\": [");
for (entry_position, entry) in entries.iter().enumerate() {
if entry_position > 0 {
out.push(',');
}
out.push_str("\n ");
write_semantics_entry(&mut out, entry);
}
if entries.is_empty() {
out.push_str("]\n }");
} else {
out.push_str("\n ]\n }");
}
}
if grammars.is_empty() {
out.push_str("]\n}\n");
} else {
out.push_str("\n ]\n}\n");
}
out
}
fn write_grammar_option_entry(out: &mut String, entry: &GrammarOptionEntry) {
out.push('{');
let _ = write!(out, "\"name\": {}", json_string(&entry.key));
let _ = write!(out, ", \"value\": {}", json_string(&entry.value));
let _ = write!(out, ", \"line\": {}", entry.line);
let _ = write!(out, ", \"column\": {}", entry.column);
let _ = write!(
out,
", \"disposition\": {}",
json_string(entry.disposition.manifest_name())
);
out.push('}');
}
fn write_semantics_entry(out: &mut String, entry: &SemanticsEntry) {
out.push('{');
let _ = write!(out, "\"kind\": {}", json_string(entry.kind.manifest_name()));
let _ = write!(
out,
", \"rule\": {}",
json_optional_string(entry.rule_name.as_deref())
);
let _ = write!(
out,
", \"rule_index\": {}",
json_optional_number(entry.rule_index)
);
let _ = write!(out, ", \"index\": {}", json_optional_number(entry.index));
let _ = write!(
out,
", \"atn_state\": {}",
json_optional_number(entry.atn_state)
);
let _ = write!(out, ", \"line\": {}", json_optional_number(entry.line));
let _ = write!(out, ", \"column\": {}", json_optional_number(entry.column));
let _ = write!(
out,
", \"body\": {}",
json_optional_string(entry.body.as_deref())
);
let _ = write!(
out,
", \"disposition\": {}",
json_string(entry.disposition.manifest_name())
);
let _ = write!(
out,
", \"template\": {}",
json_optional_string(entry.template.as_deref())
);
out.push('}');
}
fn json_optional_number(value: Option<usize>) -> String {
value.map_or_else(|| "null".to_owned(), |number| number.to_string())
}
fn json_optional_string(value: Option<&str>) -> String {
value.map_or_else(|| "null".to_owned(), json_string)
}
/// Escapes a string for JSON output; the generator hand-rolls this to avoid a
/// serialization dependency in a binary meant to be vendored into pipelines.
fn json_string(value: &str) -> String {
let mut out = String::with_capacity(value.len() + 2);
out.push('"');
for ch in value.chars() {
match ch {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
control if (control as u32) < 0x20 => {
let _ = write!(out, "\\u{:04x}", control as u32);
}
other => out.push(other),
}
}
out.push('"');
out
}
fn load_sem_patterns(path: &Path) -> io::Result<SemPatternFile> {
parse_sem_patterns(&fs::read_to_string(path)?)
}
/// Removes a trailing `#` comment from one TOML line, ignoring a `#` that
/// appears inside a quoted string (basic `"..."` or literal `'...'`). A naive
/// `split_once('#')` would corrupt valid pattern lines such as
/// `match = "text == '#'"` or a `str("#")` lower expression. Basic-string escape
/// handling means a `\"` inside `"..."` does not close the string.
fn strip_toml_comment(line: &str) -> &str {
let bytes = line.as_bytes();
let mut index = 0;
let mut quote: Option<u8> = None;
while index < bytes.len() {
let byte = bytes[index];
match quote {
// Inside a basic string, a backslash escapes the next byte.
Some(b'"') if byte == b'\\' => {
index += 2;
continue;
}
Some(q) if byte == q => quote = None,
Some(_) => {}
None if byte == b'"' || byte == b'\'' => quote = Some(byte),
None if byte == b'#' => return &line[..index],
None => {}
}
index += 1;
}
line
}
fn parse_sem_patterns(input: &str) -> io::Result<SemPatternFile> {
let mut file = SemPatternFile::default();
let mut section: Option<PatternSection> = None;
let mut fields = BTreeMap::<String, String>::new();
for raw_line in input.lines().chain(std::iter::once("")) {
let line = strip_toml_comment(raw_line).trim();
if line.is_empty() {
continue;
}
if let Some(next_section) = parse_pattern_section(line) {
flush_pattern_section(&mut file, section.take(), &mut fields)?;
section = Some(next_section);
continue;
}
let (key, value) = line.split_once('=').ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid semantic pattern line {line:?}"),
)
})?;
fields.insert(key.trim().to_owned(), parse_toml_scalar(value.trim()));
}
flush_pattern_section(&mut file, section, &mut fields)?;
Ok(file)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PatternSection {
Pattern,
Helper,
Coordinate,
}
fn parse_pattern_section(line: &str) -> Option<PatternSection> {
match line {
"[[pattern]]" => Some(PatternSection::Pattern),
"[[helper]]" => Some(PatternSection::Helper),
"[[coordinate]]" => Some(PatternSection::Coordinate),
_ => None,
}
}
fn flush_pattern_section(
file: &mut SemPatternFile,
section: Option<PatternSection>,
fields: &mut BTreeMap<String, String>,
) -> io::Result<()> {
let Some(section) = section else {
return Ok(());
};
match section {
PatternSection::Pattern => {
let match_body = take_required_field(fields, "match")?;
let lower = take_required_field(fields, "lower")?;
let id = fields
.remove("id")
.unwrap_or_else(|| format!("pattern:{}", file.patterns.len()));
file.patterns.push(SemPatternRule {
id,
match_body,
lower,
});
}
PatternSection::Helper => {
let kind = fields
.remove("kind")
.map(|value| parse_coordinate_kind(&value))
.transpose()?;
let arguments = fields
.remove("arguments")
.map_or_else(|| Ok(Vec::new()), |value| parse_helper_arguments(&value))?;
// `returns` is documentation in existing pattern files. Validate
// its shape when present, but the semantic kind determines whether
// the generated method returns bool or unit.
if let Some(returns) = fields.remove("returns") {
let expected = if kind.is_none_or(|kind| {
matches!(
kind,
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate
)
}) {
"bool"
} else {
"unit"
};
if returns != expected {
let kind = kind.map_or("predicate", SemanticsKind::manifest_name);
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"semantic helper kind {kind} requires returns = {expected:?}, got {returns:?}"
),
));
}
}
file.helpers.push(SemHelperRule {
kind,
name: take_required_field(fields, "name")?,
arguments,
lower: take_required_field(fields, "lower")?,
});
}
PatternSection::Coordinate => {
file.coordinates.push(SemCoordinateOverride {
kind: parse_coordinate_kind(&take_required_field(fields, "kind")?)?,
rule: fields.remove("rule"),
index: fields
.remove("index")
.map(|value| parse_usize_field("index", &value))
.transpose()?,
atn_state: fields
.remove("atn_state")
.map(|value| parse_usize_field("atn_state", &value))
.transpose()?,
dispose: CoordinateDispose::parse(&take_required_field(fields, "dispose")?)?,
});
}
}
fields.clear();
Ok(())
}
fn parse_helper_arguments(value: &str) -> io::Result<Vec<SemanticLiteralKind>> {
let value = value.trim();
if value.is_empty() {
return Ok(Vec::new());
}
value
.split(',')
.map(|part| match part.trim() {
"string" | "str" => Ok(SemanticLiteralKind::String),
"bool" | "boolean" => Ok(SemanticLiteralKind::Bool),
"int" | "integer" => Ok(SemanticLiteralKind::Integer),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown semantic helper argument kind {other:?}"),
)),
})
.collect()
}
fn take_required_field(fields: &mut BTreeMap<String, String>, name: &str) -> io::Result<String> {
fields.remove(name).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("semantic pattern section missing {name}"),
)
})
}
fn parse_toml_scalar(value: &str) -> String {
let value = value.trim();
if let Some(body) = value
.strip_prefix('"')
.and_then(|body| body.strip_suffix('"'))
{
return unescape_toml_basic_string(body);
}
// TOML literal strings are single-quoted with no escape processing; the body
// is taken verbatim. `strip_toml_comment` already treats `'...'` as a quoted
// string, so a value like `dispose = 'assume-false'` must have its quotes
// stripped here too, or the quotes would leak into the disposition/match and
// it would be rejected or fail to match. Require length >= 2 so a lone `'`
// is not mistaken for an empty literal.
if value.len() >= 2 {
if let Some(body) = value
.strip_prefix('\'')
.and_then(|body| body.strip_suffix('\''))
{
return body.to_owned();
}
}
value.to_owned()
}
fn unescape_toml_basic_string(value: &str) -> String {
let mut out = String::with_capacity(value.len());
let mut escaped = false;
for ch in value.chars() {
if escaped {
match ch {
'"' => out.push('"'),
'\\' => out.push('\\'),
'n' => out.push('\n'),
'r' => out.push('\r'),
't' => out.push('\t'),
other => {
out.push('\\');
out.push(other);
}
}
escaped = false;
} else if ch == '\\' {
escaped = true;
} else {
out.push(ch);
}
}
if escaped {
out.push('\\');
}
out
}
fn parse_usize_field(name: &str, value: &str) -> io::Result<usize> {
value.parse::<usize>().map_err(|error| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid {name} value {value:?}: {error}"),
)
})
}
fn parse_coordinate_kind(value: &str) -> io::Result<SemanticsKind> {
match value {
"lexer-action" => Ok(SemanticsKind::LexerAction),
"lexer-predicate" => Ok(SemanticsKind::LexerPredicate),
"parser-predicate" | "predicate" => Ok(SemanticsKind::ParserPredicate),
"parser-action" | "action" => Ok(SemanticsKind::ParserAction),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown semantic coordinate kind {other}"),
)),
}
}
#[derive(Debug)]
struct Args {
roots: Vec<PathBuf>,
library_directories: Vec<PathBuf>,
out_dir: PathBuf,
require_generated_parser: bool,
allow_unsupported_lexer_actions: bool,
sem_unknown: SemUnknownPolicy,
sem_patterns: SemPatternFile,
require_full_semantics: bool,
option_hooks: BTreeSet<String>,
generate_listener: bool,
generate_visitor: bool,
/// `--actions embedded`: the grammar contains real Rust action/predicate
/// bodies (rendered through a `.test.stg`); splice them verbatim after
/// `$`-attribute translation instead of recognizing template markup.
embedded_actions: bool,
}
enum CliCommand {
Generate(Args),
Help,
Version,
}
impl Args {
fn parse() -> Result<CliCommand, String> {
let mut roots = Vec::new();
let mut library_directories = Vec::new();
let mut out_dir = None;
let mut require_generated_parser = false;
let mut allow_unsupported_lexer_actions = false;
let mut embedded_actions = false;
let mut sem_unknown = SemUnknownPolicy::default();
let mut sem_patterns = SemPatternFile::default();
let mut require_full_semantics = false;
let mut option_hooks = BTreeSet::new();
let mut generate_listener = true;
let mut generate_visitor = false;
let mut positional_only = false;
let mut iter = env::args().skip(1);
while let Some(arg) = iter.next() {
if positional_only {
roots.push(PathBuf::from(arg));
continue;
}
match arg.as_str() {
"--" => positional_only = true,
"-I" | "--lib" => {
library_directories.push(PathBuf::from(next_arg(&mut iter, &arg)?));
}
"--out-dir" => out_dir = Some(PathBuf::from(next_arg(&mut iter, "--out-dir")?)),
"--require-generated-parser" => require_generated_parser = true,
"--allow-unsupported-lexer-actions" => allow_unsupported_lexer_actions = true,
"-listener" | "--listener" => generate_listener = true,
"-no-listener" | "--no-listener" => generate_listener = false,
"-visitor" | "--visitor" => generate_visitor = true,
"-no-visitor" | "--no-visitor" => generate_visitor = false,
"--sem-patterns" => {
sem_patterns =
load_sem_patterns(&PathBuf::from(next_arg(&mut iter, "--sem-patterns")?))
.map_err(|error| format!("failed to load --sem-patterns: {error}"))?;
}
"--require-full-semantics" => require_full_semantics = true,
"--option-hook" => {
let value = next_arg(&mut iter, "--option-hook")?;
option_hooks.insert(normalize_option_hook(&value)?);
}
"--actions" => {
let value = next_arg(&mut iter, "--actions")?;
match value.as_str() {
"embedded" => embedded_actions = true,
"templates" => embedded_actions = false,
other => {
return Err(format!(
"unknown --actions mode {other:?} (expected embedded|templates)"
));
}
}
}
"--sem-unknown" => {
sem_unknown =
SemUnknownPolicy::parse_flag(&next_arg(&mut iter, "--sem-unknown")?)?;
}
"--help" | "-h" => return Ok(CliCommand::Help),
"--version" | "-V" => return Ok(CliCommand::Version),
other if other.starts_with("-I") && other.len() > 2 => {
library_directories.push(PathBuf::from(&other[2..]));
}
other if other.starts_with('-') => {
return Err(format!("unknown argument {other}\n\n{}", usage()));
}
root => roots.push(PathBuf::from(root)),
}
}
if roots.is_empty() {
return Err(format!(
"at least one grammar root is required\n\n{}",
usage()
));
}
Ok(CliCommand::Generate(Self {
roots,
library_directories,
out_dir: out_dir.unwrap_or_else(|| PathBuf::from(".")),
require_generated_parser,
allow_unsupported_lexer_actions,
sem_unknown,
sem_patterns,
require_full_semantics,
option_hooks,
generate_listener,
generate_visitor,
embedded_actions,
}))
}
}
fn next_arg(iter: &mut impl Iterator<Item = String>, flag: &str) -> Result<String, String> {
iter.next()
.ok_or_else(|| format!("{flag} requires a value\n\n{}", usage()))
}
fn usage() -> String {
"\
Usage: antlr4-rust-gen [OPTIONS] ROOT.g4...
Options:
-I, --lib DIR Add an import/token-vocabulary lookup directory
--out-dir DIR Write generated Rust files to DIR
--actions embedded|templates Select real embedded actions or template metadata
--require-generated-parser Require generated bodies for every parser rule
--allow-unsupported-lexer-actions
Ignore unsupported lexer actions
-listener, --listener Generate the typed listener and walker (default)
-no-listener, --no-listener Do not generate the typed listener or walker
-visitor, --visitor Generate the typed visitor
-no-visitor, --no-visitor Do not generate the typed visitor (default)
--sem-unknown error|hook|assume-true|assume-false
Choose unsupported semantic predicate policy
--sem-patterns FILE Load semantic helper patterns
--option-hook KEY=VALUE Acknowledge an option implemented by caller hooks
--require-full-semantics Fail if any semantic coordinate or option is unsupported
-V, --version Print version
-h, --help Print this help"
.to_owned()
}
#[derive(Clone, Debug, Default)]
struct CodegenData<'a> {
literal_names: Vec<Option<String>>,
symbolic_names: Vec<Option<String>>,
rule_names: Vec<String>,
channel_names: Vec<String>,
mode_names: Vec<String>,
lexer_atn_words: Vec<i32>,
lexer_atn: Option<LexerAtn>,
parser_atn: Option<ParserAtn>,
lexer_dfa_words: Vec<u32>,
semantic: Option<&'a SemanticGrammar>,
graph: Option<&'a FinalizedAtnGraph>,
provenance: Option<&'a ProvenanceIndex>,
sources: Option<&'a SourceSet>,
}
impl<'a> CodegenData<'a> {
fn from_lexer(compiled: &'a CompiledLexer, sources: &'a SourceSet) -> Self {
let recognizer = &compiled.semantic.recognizer;
Self {
literal_names: recognizer.literal_names.clone(),
symbolic_names: recognizer.symbolic_names.clone(),
rule_names: recognizer.rule_names.clone(),
channel_names: recognizer
.channel_names
.iter()
.map(|name| name.clone().unwrap_or_default())
.collect(),
mode_names: recognizer.mode_names.clone(),
lexer_atn_words: compiled.runtime_artifact.atn_words.clone(),
lexer_atn: Some(compiled.atn.clone()),
parser_atn: None,
lexer_dfa_words: compiled.runtime_artifact.dfa_words.clone(),
semantic: Some(&compiled.semantic),
graph: Some(&compiled.graph),
provenance: Some(&compiled.provenance),
sources: Some(sources),
}
}
fn from_parser(compiled: &'a CompiledParser, sources: &'a SourceSet) -> Self {
let recognizer = &compiled.semantic.recognizer;
Self {
literal_names: recognizer.literal_names.clone(),
symbolic_names: recognizer.symbolic_names.clone(),
rule_names: recognizer.rule_names.clone(),
channel_names: recognizer
.channel_names
.iter()
.map(|name| name.clone().unwrap_or_default())
.collect(),
mode_names: recognizer.mode_names.clone(),
lexer_atn_words: Vec::new(),
lexer_atn: None,
parser_atn: Some(compiled.packed.clone()),
lexer_dfa_words: Vec::new(),
semantic: Some(&compiled.semantic),
graph: Some(&compiled.graph),
provenance: Some(&compiled.provenance),
sources: Some(sources),
}
}
fn lexer_atn(&self) -> io::Result<&LexerAtn> {
self.lexer_atn.as_ref().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"direct lexer artifact is unavailable",
)
})
}
fn parser_atn(&self) -> io::Result<&ParserAtn> {
self.parser_atn.as_ref().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"direct parser artifact is unavailable",
)
})
}
}
#[derive(Clone, Copy, Debug)]
struct StructuralElement<'a> {
element: &'a Element,
}
#[derive(Clone, Debug)]
struct StructuralAction {
rule_index: usize,
action_index: usize,
state: usize,
body: String,
span: SourceSpan,
authored: bool,
}
#[derive(Clone, Debug)]
struct StructuralPredicate {
rule_index: usize,
predicate_index: usize,
body: String,
fail: Option<String>,
span: SourceSpan,
}
#[derive(Clone, Debug)]
struct StructuralRuleCall {
target_rule_index: usize,
state: usize,
arguments: Option<String>,
}
fn structural_elements<'model>(
data: &CodegenData<'model>,
) -> io::Result<Vec<StructuralElement<'model>>> {
let semantic = data.semantic.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"structural grammar model is unavailable",
)
})?;
let mut elements = Vec::new();
for rule in &semantic.unit.rules {
for alternative in &rule.block.alternatives {
collect_structural_elements(alternative, &mut elements);
}
}
Ok(elements)
}
fn collect_structural_elements<'a>(
alternative: &'a Alternative,
elements: &mut Vec<StructuralElement<'a>>,
) {
for element in &alternative.elements {
elements.push(StructuralElement { element });
if let ElementKind::Block(block) = &element.kind {
for nested in &block.alternatives {
collect_structural_elements(nested, elements);
}
}
}
}
fn structural_actions(data: &CodegenData<'_>) -> io::Result<Vec<StructuralAction>> {
let semantic = data.semantic.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"structural grammar model is unavailable",
)
})?;
let graph = data.graph.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"finalized ATN graph is unavailable",
)
})?;
let provenance = data
.provenance
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "provenance is unavailable"))?;
let mut actions = Vec::new();
for item in structural_elements(data)? {
let ElementKind::Action { id, body } = &item.element.kind else {
continue;
};
let binding = semantic
.bindings
.actions
.get(id)
.expect("semantic action binding exists");
let authored = provenance
.origins(ModelNodeId::Element(item.element.id))
.iter()
.any(|origin| matches!(origin, Origin::Authored { .. }));
for transition in graph.transitions_for_model(ModelNodeId::Element(item.element.id)) {
let FinalizedTransitionKind::Action { rule_index, .. } = transition.kind else {
continue;
};
actions.push(StructuralAction {
rule_index,
action_index: binding.index,
state: transition.source,
body: body.clone(),
span: item.element.span.clone(),
authored,
});
}
}
actions.sort_by_key(|action| (action.state, action.rule_index, action.action_index));
actions.dedup_by_key(|action| action.state);
Ok(actions)
}
fn structural_predicates(data: &CodegenData<'_>) -> io::Result<Vec<StructuralPredicate>> {
let semantic = data.semantic.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"structural grammar model is unavailable",
)
})?;
let graph = data.graph.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"finalized ATN graph is unavailable",
)
})?;
let mut predicates = Vec::new();
for item in structural_elements(data)? {
let ElementKind::Predicate { id, body, fail, .. } = &item.element.kind else {
continue;
};
let binding = semantic
.bindings
.predicates
.get(id)
.expect("semantic predicate binding exists");
for transition in graph.transitions_for_model(ModelNodeId::Element(item.element.id)) {
let FinalizedTransitionKind::Predicate {
rule_index,
predicate_index,
..
} = transition.kind
else {
continue;
};
debug_assert_eq!(binding.index, predicate_index);
predicates.push(StructuralPredicate {
rule_index,
predicate_index,
body: body.clone(),
fail: fail.clone(),
span: item.element.span.clone(),
});
}
}
predicates.sort_by_key(|predicate| (predicate.rule_index, predicate.predicate_index));
predicates.dedup_by_key(|predicate| (predicate.rule_index, predicate.predicate_index));
Ok(predicates)
}
fn structural_rule_calls(data: &CodegenData<'_>) -> io::Result<Vec<StructuralRuleCall>> {
let semantic = data.semantic.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"structural grammar model is unavailable",
)
})?;
let graph = data.graph.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"finalized ATN graph is unavailable",
)
})?;
let mut calls = Vec::new();
for item in structural_elements(data)? {
let ElementKind::RuleCall(call) = &item.element.kind else {
continue;
};
let binding = semantic
.bindings
.rule_calls
.get(&item.element.id)
.expect("semantic rule-call binding exists");
for transition in graph.transitions_for_model(ModelNodeId::Element(item.element.id)) {
let FinalizedTransitionKind::Rule { rule_index, .. } = transition.kind else {
continue;
};
debug_assert_eq!(
semantic.recognizer.rule_numbers[&binding.target],
rule_index
);
calls.push(StructuralRuleCall {
target_rule_index: rule_index,
state: transition.source,
arguments: call.arguments.clone(),
});
}
}
calls.sort_by_key(|call| (call.state, call.target_rule_index));
calls.dedup_by_key(|call| call.state);
Ok(calls)
}
fn structural_line_column(data: &CodegenData<'_>, span: &SourceSpan) -> (usize, usize) {
data.sources
.and_then(|sources| sources.get(span.source))
.and_then(|source| source.line_column(span.bytes.start))
.unwrap_or((0, 0))
}
fn structural_embedded_model(
data: &CodegenData<'_>,
include_members: bool,
) -> io::Result<embedded::EmbeddedModel> {
let semantic = data.semantic.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"structural grammar model is unavailable",
)
})?;
let mut rules = data
.rule_names
.iter()
.map(|name| embedded::RuleModel {
name: name.clone(),
..embedded::RuleModel::default()
})
.collect::<Vec<_>>();
for rule in &semantic.unit.rules {
let Some(&rule_index) = semantic.recognizer.rule_numbers.get(&rule.id) else {
continue;
};
let attributes = semantic
.bindings
.attributes
.get(&rule.id)
.cloned()
.unwrap_or_default();
let attrs = attributes
.arguments
.iter()
.chain(&attributes.returns)
.chain(&attributes.locals)
.map(structural_attr_decl)
.collect();
let init_body = rule
.actions
.iter()
.find(|action| action.name == "init")
.map(|action| action.body.clone());
let after_body = rule
.actions
.iter()
.find(|action| action.name == "after")
.map(|action| action.body.clone());
rules[rule_index] = embedded::RuleModel {
name: rule.name.clone(),
attrs,
local_names: attributes
.locals
.iter()
.map(|attribute| attribute.name.clone())
.collect(),
arg_names: attributes
.arguments
.iter()
.map(|attribute| attribute.name.clone())
.collect(),
init_body,
after_body,
alts: structural_rule_alternatives(rule, &semantic.recognizer.vocabulary),
};
}
let mut parser_members = embedded::MembersModel::default();
if include_members {
for action in &semantic.unit.actions {
if action.name == "members"
&& action
.scope
.as_deref()
.is_none_or(|scope| scope == "parser")
{
embedded::classify_members(&action.body, &mut parser_members)?;
}
}
}
Ok(embedded::EmbeddedModel {
rules,
parser_members,
})
}
fn structural_attr_decl(attribute: &AttributeSymbol) -> embedded::AttrDecl {
embedded::AttrDecl {
name: attribute.name.clone(),
ty: embedded::map_attr_type(&attribute.ty),
}
}
fn structural_rule_alternatives(rule: &Rule, vocabulary: &Vocabulary) -> Vec<embedded::AltModel> {
let Some(left_recursion) = &rule.left_recursion else {
return rule
.block
.alternatives
.iter()
.map(|alternative| structural_alt_model(alternative, None, vocabulary))
.collect();
};
left_recursion
.original_to_rewritten
.iter()
.filter_map(|(original, rewritten)| {
let alternative = find_alternative(&rule.block, *rewritten)?;
let leading_ref = left_recursion
.alternative_kinds
.get(original)
.is_some_and(|kind| {
matches!(
kind,
LeftRecursiveAlternativeKind::Binary | LeftRecursiveAlternativeKind::Suffix
)
})
.then(|| {
let removed = left_recursion
.deleted_labels
.values()
.find(|removed| removed.original_alternative == *original);
embedded::ElementRef {
label: removed.map(|removed| removed.label.name.clone()),
target: removed
.map_or_else(|| rule.name.clone(), |removed| removed.target.clone()),
token_types: Vec::new(),
is_block: false,
is_list: removed
.is_some_and(|removed| removed.label.kind == LabelKind::List),
cardinality: embedded::ChildCardinality::ONE,
stable_accessor: true,
}
});
Some(structural_alt_model(alternative, leading_ref, vocabulary))
})
.collect()
}
fn find_alternative(block: &Block, id: AlternativeId) -> Option<&Alternative> {
for alternative in &block.alternatives {
if alternative.id == id {
return Some(alternative);
}
for element in &alternative.elements {
if let ElementKind::Block(nested) = &element.kind
&& let Some(found) = find_alternative(nested, id)
{
return Some(found);
}
}
}
None
}
fn structural_alt_model(
alternative: &Alternative,
removed_leading_ref: Option<embedded::ElementRef>,
vocabulary: &Vocabulary,
) -> embedded::AltModel {
let leading_target = removed_leading_ref
.as_ref()
.map(|element| element.target.clone());
let mut children = structural_context_children(&alternative.elements, vocabulary);
if let Some(leading) = &removed_leading_ref {
add_child_cardinality(
&mut children,
&leading.target,
embedded::ChildCardinality::ONE,
);
}
let mut refs = removed_leading_ref.into_iter().collect();
collect_structural_context_refs(&alternative.elements, &mut refs, true, vocabulary);
embedded::AltModel {
label: alternative.label.as_ref().map(|label| label.value.clone()),
span: (
usize::try_from(alternative.span.bytes.start).expect("source offset exceeds usize"),
usize::try_from(alternative.span.bytes.end).expect("source offset exceeds usize"),
),
refs,
children,
leading_target: leading_target.or_else(|| {
alternative
.elements
.first()
.and_then(structural_leading_target)
}),
}
}
fn collect_structural_context_refs(
elements: &[Element],
refs: &mut Vec<embedded::ElementRef>,
stable_accessor: bool,
vocabulary: &Vocabulary,
) {
collect_structural_context_refs_with_cardinality(
elements,
refs,
stable_accessor,
embedded::ChildCardinality::ONE,
vocabulary,
);
}
fn collect_structural_context_refs_with_cardinality(
elements: &[Element],
refs: &mut Vec<embedded::ElementRef>,
stable_accessor: bool,
enclosing_cardinality: embedded::ChildCardinality,
vocabulary: &Vocabulary,
) {
for element in elements {
let label = element.label.as_ref().map(|label| label.name.clone());
let is_list = element
.label
.as_ref()
.is_some_and(|label| label.kind == LabelKind::List);
let cardinality = multiply_child_cardinalities(
enclosing_cardinality,
quantified_cardinality(embedded::ChildCardinality::ONE, element.quantifier),
);
match &element.kind {
ElementKind::RuleCall(call) => refs.push(embedded::ElementRef {
label,
target: call.name.clone(),
token_types: Vec::new(),
is_block: false,
is_list,
cardinality,
stable_accessor,
}),
ElementKind::Terminal(terminal) => {
refs.push(embedded::ElementRef {
label,
target: structural_terminal_target(terminal),
token_types: structural_terminal_token_types(terminal, vocabulary),
is_block: !matches!(terminal, Terminal::Token(_)),
is_list,
cardinality,
stable_accessor,
});
}
ElementKind::Block(block) => {
let token_types = structural_block_token_types(block, vocabulary);
if !token_types.is_empty() {
refs.push(embedded::ElementRef {
label,
target: String::new(),
token_types,
is_block: true,
is_list,
cardinality,
stable_accessor,
});
continue;
}
if label.is_some() {
refs.push(embedded::ElementRef {
label,
target: String::new(),
token_types: Vec::new(),
is_block: true,
is_list,
cardinality,
stable_accessor: false,
});
}
let nested_stable = stable_accessor && block.alternatives.len() == 1;
for alternative in &block.alternatives {
collect_structural_context_refs_with_cardinality(
&alternative.elements,
refs,
nested_stable,
cardinality,
vocabulary,
);
}
}
ElementKind::Set { inverted, elements } => {
refs.push(embedded::ElementRef {
label,
target: String::new(),
token_types: structural_set_token_types(*inverted, elements, vocabulary),
is_block: true,
is_list,
cardinality,
stable_accessor,
});
}
ElementKind::Range(..) if label.is_some() => refs.push(embedded::ElementRef {
label,
target: String::new(),
token_types: Vec::new(),
is_block: false,
is_list,
cardinality,
stable_accessor: false,
}),
ElementKind::Range(..)
| ElementKind::Action { .. }
| ElementKind::Predicate { .. }
| ElementKind::Epsilon => {}
}
}
}
fn structural_terminal_target(terminal: &Terminal) -> String {
match terminal {
Terminal::Token(name) | Terminal::Literal(name) | Terminal::LexerCharSet(name) => {
name.clone()
}
Terminal::Eof => "EOF".to_owned(),
Terminal::Wildcard => String::new(),
}
}
fn structural_terminal_token_types(terminal: &Terminal, vocabulary: &Vocabulary) -> Vec<i32> {
if matches!(terminal, Terminal::Wildcard) {
return (1..=vocabulary.max_token_type()).collect();
}
let token_type = match terminal {
Terminal::Token(name) => vocabulary.by_name.get(name).copied(),
Terminal::Literal(literal) => vocabulary.by_literal.get(literal).copied(),
Terminal::Eof => Some(TOKEN_EOF),
Terminal::LexerCharSet(_) | Terminal::Wildcard => None,
};
token_type.into_iter().collect()
}
fn structural_set_token_types(
inverted: bool,
elements: &[SetElement],
vocabulary: &Vocabulary,
) -> Vec<i32> {
let mut members = BTreeSet::new();
for element in elements {
match element {
SetElement::Terminal { value, .. } => {
members.extend(structural_terminal_token_types(value, vocabulary));
}
SetElement::Range { start, stop, .. } => {
let Some(start) = vocabulary.by_literal.get(start).copied() else {
continue;
};
let Some(stop) = vocabulary.by_literal.get(stop).copied() else {
continue;
};
if start <= stop {
members.extend(start..=stop);
}
}
}
}
if inverted {
(1..=vocabulary.max_token_type())
.filter(|token_type| !members.contains(token_type))
.collect()
} else {
members.into_iter().collect()
}
}
fn structural_element_token_types(element: &Element, vocabulary: &Vocabulary) -> Vec<i32> {
match &element.kind {
ElementKind::Terminal(terminal) => structural_terminal_token_types(terminal, vocabulary),
ElementKind::Set { inverted, elements } => {
structural_set_token_types(*inverted, elements, vocabulary)
}
ElementKind::Block(block) => structural_block_token_types(block, vocabulary),
ElementKind::RuleCall(_)
| ElementKind::Range(..)
| ElementKind::Action { .. }
| ElementKind::Predicate { .. }
| ElementKind::Epsilon => Vec::new(),
}
}
fn structural_block_token_types(block: &Block, vocabulary: &Vocabulary) -> Vec<i32> {
let mut token_types = BTreeSet::new();
for alternative in &block.alternatives {
let mut elements = alternative.elements.iter().filter(|element| {
!matches!(
element.kind,
ElementKind::Action { .. } | ElementKind::Predicate { .. } | ElementKind::Epsilon
)
});
let Some(element) = elements.next() else {
return Vec::new();
};
if elements.next().is_some() || element.quantifier != Quantifier::One {
return Vec::new();
}
let alternative_types = structural_element_token_types(element, vocabulary);
if alternative_types.is_empty() {
return Vec::new();
}
token_types.extend(alternative_types);
}
token_types.into_iter().collect()
}
fn structural_terminal_child_target(
terminal: &Terminal,
vocabulary: &Vocabulary,
) -> Option<String> {
match terminal {
Terminal::Token(name) => Some(name.clone()),
Terminal::Literal(literal) => {
let token_type = vocabulary.by_literal.get(literal)?;
vocabulary
.tokens
.iter()
.find(|token| token.number == *token_type)
.and_then(|token| token.name.as_ref())
.filter(|name| !name.starts_with("T__"))
.cloned()
}
Terminal::Eof => Some("EOF".to_owned()),
Terminal::LexerCharSet(_) | Terminal::Wildcard => None,
}
}
fn structural_context_children(
elements: &[Element],
vocabulary: &Vocabulary,
) -> BTreeMap<String, embedded::ChildCardinality> {
let mut children = BTreeMap::new();
for element in elements {
let mut element_children = match &element.kind {
ElementKind::RuleCall(call) => {
BTreeMap::from([(call.name.clone(), embedded::ChildCardinality::ONE)])
}
ElementKind::Terminal(terminal) => {
structural_terminal_child_target(terminal, vocabulary)
.map(|target| BTreeMap::from([(target, embedded::ChildCardinality::ONE)]))
.unwrap_or_default()
}
ElementKind::Block(block) => structural_block_children(block, vocabulary),
ElementKind::Range(..)
| ElementKind::Set { .. }
| ElementKind::Action { .. }
| ElementKind::Predicate { .. }
| ElementKind::Epsilon => BTreeMap::new(),
};
for cardinality in element_children.values_mut() {
*cardinality = quantified_cardinality(*cardinality, element.quantifier);
}
for (target, cardinality) in element_children {
add_child_cardinality(&mut children, &target, cardinality);
}
}
children
}
fn structural_block_children(
block: &Block,
vocabulary: &Vocabulary,
) -> BTreeMap<String, embedded::ChildCardinality> {
choice_child_cardinalities(
block
.alternatives
.iter()
.map(|alternative| structural_context_children(&alternative.elements, vocabulary)),
)
}
fn choice_child_cardinalities(
alternatives: impl IntoIterator<Item = BTreeMap<String, embedded::ChildCardinality>>,
) -> BTreeMap<String, embedded::ChildCardinality> {
let alternatives = alternatives.into_iter().collect::<Vec<_>>();
let targets = alternatives
.iter()
.flat_map(|alternative| alternative.keys().cloned())
.collect::<BTreeSet<_>>();
targets
.into_iter()
.map(|target| {
let mut min = usize::MAX;
let mut max = Some(0_usize);
for alternative in &alternatives {
let cardinality = alternative
.get(&target)
.copied()
.unwrap_or(embedded::ChildCardinality::ZERO);
min = min.min(cardinality.min);
max = match (max, cardinality.max) {
(Some(current), Some(next)) => Some(current.max(next)),
_ => None,
};
}
(
target,
embedded::ChildCardinality {
min: if min == usize::MAX { 0 } else { min },
max,
},
)
})
.collect()
}
fn add_child_cardinality(
children: &mut BTreeMap<String, embedded::ChildCardinality>,
target: &str,
cardinality: embedded::ChildCardinality,
) {
let total = children
.entry(target.to_owned())
.or_insert(embedded::ChildCardinality::ZERO);
total.min = total.min.saturating_add(cardinality.min);
total.max = match (total.max, cardinality.max) {
(Some(current), Some(next)) => Some(current.saturating_add(next)),
_ => None,
};
}
fn quantified_cardinality(
cardinality: embedded::ChildCardinality,
quantifier: Quantifier,
) -> embedded::ChildCardinality {
match quantifier {
Quantifier::One => cardinality,
Quantifier::Optional { .. } => embedded::ChildCardinality {
min: 0,
max: cardinality.max,
},
Quantifier::ZeroOrMore { .. } => embedded::ChildCardinality {
min: 0,
max: (cardinality.max == Some(0)).then_some(0),
},
Quantifier::OneOrMore { .. } => embedded::ChildCardinality {
min: cardinality.min,
max: (cardinality.max == Some(0)).then_some(0),
},
}
}
const fn multiply_child_cardinalities(
left: embedded::ChildCardinality,
right: embedded::ChildCardinality,
) -> embedded::ChildCardinality {
let max = match (left.max, right.max) {
(Some(0), _) | (_, Some(0)) => Some(0),
(Some(left), Some(right)) => Some(left.saturating_mul(right)),
_ => None,
};
embedded::ChildCardinality {
min: left.min.saturating_mul(right.min),
max,
}
}
fn structural_leading_target(element: &Element) -> Option<String> {
match &element.kind {
ElementKind::RuleCall(call) => Some(call.name.clone()),
ElementKind::Terminal(Terminal::Token(name)) => Some(name.clone()),
_ => None,
}
}
/// Sets the rendered template for a lexer predicate coordinate, replacing any
/// existing (translated) entry so a per-coordinate override wins over a
/// built-in translation, or appending a new entry for an uncovered coordinate.
fn set_lexer_predicate_template(
predicates: &mut Vec<((usize, usize), PredicateTemplate)>,
coordinate: (usize, usize),
template: PredicateTemplate,
) {
if let Some(entry) = predicates.iter_mut().find(|(pred, _)| *pred == coordinate) {
entry.1 = template;
} else {
predicates.push((coordinate, template));
}
}
/// Renders a Rust lexer module that delegates token recognition to the shared
/// ATN interpreter.
///
/// The emitted lexer owns only generated metadata and a `BaseLexer`. Keeping
/// recognition in the runtime avoids emitting thousands of lines of
/// grammar-specific Rust control flow for the first target implementation.
#[allow(clippy::too_many_arguments)]
fn render_lexer(
grammar_name: &str,
data: &CodegenData<'_>,
allow_unsupported_lexer_actions: bool,
sem_unknown: SemUnknownPolicy,
patterns: &SemPatternFile,
embedded: bool,
) -> io::Result<String> {
let type_name = rust_type_name(grammar_name);
let metadata = render_metadata(grammar_name, data);
let token_constants = render_token_constants(data);
// Embedded mode: lexer action/predicate bodies are verbatim Rust from the
// rendered grammar; translate the recognizer surface textually and skip
// the template machinery entirely.
let embedded_lexer_actions = if embedded {
structural_embedded_lexer_actions(data)?
} else {
Vec::new()
};
let embedded_lexer_predicates = if embedded {
structural_embedded_lexer_predicates(data)?
} else {
Vec::new()
};
let mut actions = if embedded {
Vec::new()
} else {
let actions = structural_lexer_action_templates(data, patterns)?;
reject_unsupported_lexer_action_templates(
&actions
.iter()
.map(|(_, template)| template.clone())
.collect::<Vec<_>>(),
allow_unsupported_lexer_actions,
)?;
actions
};
// Any per-coordinate override replaces a translated action. Hook overrides
// fall through to the semantic hook object; assume-* overrides are no-ops.
actions.retain(|((rule_index, action_index), _)| {
let rule_name = usize::try_from(*rule_index)
.ok()
.and_then(|rule| data.rule_names.get(rule).map(String::as_str));
patterns
.coordinate_override(
SemanticsKind::LexerAction,
rule_name,
usize::try_from(*action_index).ok(),
None,
)
.is_none()
});
let mut predicates = if embedded {
Vec::new()
} else {
structural_predicate_templates(data, SemanticsKind::LexerPredicate, patterns)?
};
// Apply per-coordinate overrides to every lexer predicate transition. Hook
// coordinates are represented explicitly so generated dispatch can fall
// through to the caller-owned semantic hook object.
for coordinate in lexer_predicate_transitions(data)? {
let (rule_index, pred_index) = coordinate;
let dispose = patterns
.coordinate_override(
SemanticsKind::LexerPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
None,
)
.map(|override_| override_.dispose);
let covered = predicates.iter().any(|(pred, _)| *pred == coordinate);
match dispose {
Some(CoordinateDispose::Hook) => {
set_lexer_predicate_template(&mut predicates, coordinate, PredicateTemplate::Hook);
}
Some(CoordinateDispose::Error) => {}
Some(CoordinateDispose::AssumeFalse) => {
set_lexer_predicate_template(&mut predicates, coordinate, PredicateTemplate::False);
}
Some(CoordinateDispose::AssumeTrue) => {
set_lexer_predicate_template(&mut predicates, coordinate, PredicateTemplate::True);
}
None => {
if !covered && sem_unknown == SemUnknownPolicy::Hook {
set_lexer_predicate_template(
&mut predicates,
coordinate,
PredicateTemplate::Hook,
);
}
}
}
}
// Predicate coordinates with no translated template normally fall back to
// always-true; `--sem-unknown=assume-false` flips that fallback, which
// requires the hook-taking token path even when no dispatch is generated.
let unknown_predicates_assume_false = sem_unknown == SemUnknownPolicy::AssumeFalse
&& predicates.is_empty()
&& !lexer_predicate_transitions(data)?.is_empty();
let lexer_dfa_data = compiled_lexer_dfa_words(data);
let lexer_typed_hook_mappings = if embedded {
Vec::new()
} else {
lexer_typed_hook_mappings(data, patterns, &actions)?
};
let typed_hook_adapter =
render_lexer_typed_hook_adapter(&type_name, &lexer_typed_hook_mappings);
let typed_lexer_constructor = if lexer_typed_hook_mappings.is_empty() {
String::new()
} else {
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
format!(
"impl<I, T> {type_name}<I, {adapter_name}<T>>\nwhere\n I: CharStream,\n T: {trait_name},\n{{\n pub fn with_typed_hooks(input: I, hooks: T) -> Self {{\n Self::with_hooks(input, {adapter_name}::new(hooks))\n }}\n}}\n"
)
};
let action_coordinates = lexer_custom_actions(data)?;
let has_hook_disposed_actions = lexer_actions_require_semantic_hooks(
&action_coordinates,
&data.rule_names,
&actions,
patterns,
sem_unknown,
);
let has_semantic_hooks = has_hook_disposed_actions
|| !lexer_typed_hook_mappings.is_empty()
|| actions
.iter()
.any(|(_, template)| matches!(template, ActionTemplate::Hook(_)))
|| predicates
.iter()
.any(|(_, template)| matches!(template, PredicateTemplate::Hook));
let has_action_dispatch = if embedded {
embedded_lexer_actions
.iter()
.any(|(_, statement)| !statement.trim().is_empty())
} else {
lexer_actions_need_dispatch(&actions)
};
let action_method = if embedded {
render_embedded_lexer_action_method(&embedded_lexer_actions)
} else {
render_lexer_action_method(&actions)
};
let predicate_method = if embedded {
render_embedded_lexer_predicate_method(&embedded_lexer_predicates)
} else {
render_lexer_predicate_method(&predicates, sem_unknown)
};
let has_predicate_dispatch = if embedded {
!embedded_lexer_predicates.is_empty()
} else {
!predicates.is_empty()
};
let lexer_unknown_policy = match sem_unknown {
SemUnknownPolicy::AssumeTrue => "antlr4_runtime::UnknownSemanticPolicy::AssumeTrue",
SemUnknownPolicy::AssumeFalse => "antlr4_runtime::UnknownSemanticPolicy::AssumeFalse",
SemUnknownPolicy::Hook | SemUnknownPolicy::Error => {
"antlr4_runtime::UnknownSemanticPolicy::Error"
}
};
let semantic_action = if has_action_dispatch {
"Self::run_action"
} else {
"|_, _| false"
};
let semantic_predicate = if has_predicate_dispatch {
"Self::run_predicate"
} else {
"|_, _| None"
};
let lifecycle_next_token_call = format!(
"antlr4_runtime::atn::lexer::next_token_compiled_with_semantic_dispatch(&mut self.base, sink, atn(), lexer_dfa(), &mut self.hooks, {semantic_action}, {semantic_predicate}, {lexer_unknown_policy}, |_, _, _| {{}})"
);
let next_token_call = if has_semantic_hooks {
lifecycle_next_token_call.clone()
} else if !has_action_dispatch && !has_predicate_dispatch && !unknown_predicates_assume_false {
"antlr4_runtime::atn::lexer::next_token_compiled(&mut self.base, sink, atn(), lexer_dfa())"
.to_owned()
} else {
let action = if has_action_dispatch {
"|base, action| { let _ = Self::run_action(base, action); }"
} else {
"|_, _| {}"
};
let predicate = if has_predicate_dispatch {
"|base, predicate| Self::run_predicate(base, predicate).unwrap_or(true)"
} else if unknown_predicates_assume_false {
"|_, _| false"
} else {
"|_, _| true"
};
format!(
"antlr4_runtime::atn::lexer::next_token_compiled_with_hooks(&mut self.base, sink, atn(), lexer_dfa(), {action}, {predicate}, |_, _, _| {{}})"
)
};
let next_token_call = if has_semantic_hooks {
next_token_call
} else {
format!(
"if H::ENABLES_LEXER_LIFECYCLE {{\n {lifecycle_next_token_call}\n }} else {{\n {next_token_call}\n }}"
)
};
let generated_header = GENERATED_MODULE_HEADER;
let generated_footer = GENERATED_MODULE_FOOTER;
Ok(format!(
r#"{generated_header}use antlr4_runtime::char_stream::CharStream;
use antlr4_runtime::recognizer::RecognizerData;
use antlr4_runtime::token::{{TokenId, TokenSink, TokenSource, TokenStoreError}};
use antlr4_runtime::atn::LexerAtn;
use antlr4_runtime::atn::lexer_dfa::CompiledLexerDfa;
use antlr4_runtime::atn::serialized::AtnDeserializer;
use antlr4_runtime::{{BaseLexer, GeneratedLexer, GrammarMetadata, Lexer, Recognizer}};
use std::sync::OnceLock;
{token_constants}
{metadata}
{typed_hook_adapter}
static ATN_CELL: OnceLock<LexerAtn> = OnceLock::new();
/// Deserializes and caches the grammar ATN for all lexer instances.
fn atn() -> &'static LexerAtn {{
ATN_CELL.get_or_init(|| {{
let serialized = metadata().serialized_atn();
AtnDeserializer::new(&serialized)
.deserialize()
.expect("generated lexer contains a valid ANTLR serialized ATN")
}})
}}
static LEXER_DFA_DATA: &[u32] = &[{lexer_dfa_data}];
static LEXER_DFA_CELL: OnceLock<CompiledLexerDfa> = OnceLock::new();
/// Ahead-of-time lexer DFA tables compiled by antlr4-rust-gen, embedded so
/// runtime startup only deserializes them. Rebuilt from the ATN instead when
/// the embedded stream comes from a different runtime version.
fn lexer_dfa() -> &'static CompiledLexerDfa {{
LEXER_DFA_CELL.get_or_init(|| {{
CompiledLexerDfa::from_serialized(LEXER_DFA_DATA)
.unwrap_or_else(|| CompiledLexerDfa::compile(atn()))
}})
}}
#[derive(Clone, Debug)]
pub struct {type_name}<I, H = antlr4_runtime::NoSemanticHooks>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
base: BaseLexer<I>,
hooks: H,
}}
impl<I> {type_name}<I, antlr4_runtime::NoSemanticHooks>
where
I: CharStream,
{{
pub fn new(input: I) -> Self {{
Self::with_hooks(input, antlr4_runtime::NoSemanticHooks)
}}
}}
impl<I, H> {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
pub fn with_hooks(input: I, hooks: H) -> Self {{
let grammar_metadata = metadata();
let data = RecognizerData::new(
grammar_metadata.grammar_file_name(),
grammar_metadata.vocabulary(),
)
.with_rule_names(grammar_metadata.rule_names().iter().copied())
.with_channel_names(grammar_metadata.channel_names().iter().copied())
.with_mode_names(grammar_metadata.mode_names().iter().copied());
Self {{ base: BaseLexer::new(input, data).with_shared_dfa(atn()), hooks }}
}}
pub fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
/// Adds a listener for lexer diagnostics.
pub fn add_error_listener<T>(&mut self, listener: T)
where
T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,
{{
self.base.add_error_listener(listener);
}}
/// Removes every lexer error listener, including the default console listener.
pub fn remove_error_listeners(&mut self) {{
self.base.remove_error_listeners();
}}
/// Routes every token through ATN interpretation instead of the compiled
/// lexer DFA, so the learned-DFA trace (`lexer_dfa_string`) observes each
/// match.
pub fn set_force_interpreted(&mut self, force_interpreted: bool) {{
self.base.set_force_interpreted(force_interpreted);
}}
/// Resets this lexer and any caller-owned lifecycle state for reuse.
pub fn reset(&mut self) {{
if H::ENABLES_LEXER_LIFECYCLE {{
antlr4_runtime::atn::lexer::reset_with_semantic_hooks(
&mut self.base,
&mut self.hooks,
);
}} else {{
self.base.reset();
}}
}}
/// Replaces the input stream and resets runtime and lifecycle state.
pub fn set_input_stream(&mut self, input: I) {{
if H::ENABLES_LEXER_LIFECYCLE {{
antlr4_runtime::atn::lexer::set_input_stream_with_semantic_hooks(
&mut self.base,
&mut self.hooks,
input,
);
}} else {{
self.base.set_input_stream(input);
}}
}}
/// Clears the learned lexer DFA shared by this grammar.
pub fn clear_dfa(&self) {{
self.base.clear_dfa();
}}
{action_method}
{predicate_method}
}}
{typed_lexer_constructor}
impl<I, H> GeneratedLexer for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
}}
impl<I, H> Recognizer for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn data(&self) -> &antlr4_runtime::RecognizerData {{
self.base.data()
}}
fn data_mut(&mut self) -> &mut antlr4_runtime::RecognizerData {{
self.base.data_mut()
}}
}}
impl<I, H> Lexer for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn mode(&self) -> i32 {{ self.base.mode() }}
fn set_mode(&mut self, mode: i32) {{ self.base.set_mode(mode); }}
fn push_mode(&mut self, mode: i32) {{ self.base.push_mode(mode); }}
fn pop_mode(&mut self) -> Option<i32> {{ self.base.pop_mode() }}
}}
impl<I, H> TokenSource for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result<TokenId, TokenStoreError> {{
{next_token_call}
}}
fn line(&self) -> usize {{ self.base.line() }}
fn column(&self) -> usize {{ self.base.column() }}
fn source_name(&self) -> &str {{ self.base.source_name() }}
fn source_text(&self) -> Option<std::rc::Rc<str>> {{ self.base.source_text() }}
fn drain_errors(&mut self) -> Vec<antlr4_runtime::token::TokenSourceError> {{
self.base.drain_errors()
}}
fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool {{
antlr4_runtime::Recognizer::notify_error_listeners(
self,
source_error.line,
source_error.column,
&source_error.message,
None,
);
true
}}
fn lexer_dfa_string(&self) -> String {{
self.base.lexer_dfa_string()
}}
}}
{generated_footer}"#
))
}
fn lexer_actions_require_semantic_hooks(
coordinates: &[(i32, i32)],
rule_names: &[String],
actions: &[((i32, i32), ActionTemplate)],
patterns: &SemPatternFile,
sem_unknown: SemUnknownPolicy,
) -> bool {
coordinates.iter().any(|coordinate| {
let (rule_index, action_index) = *coordinate;
let rule_name = usize::try_from(rule_index)
.ok()
.and_then(|rule| rule_names.get(rule).map(String::as_str));
if let Some(override_) = patterns.coordinate_override(
SemanticsKind::LexerAction,
rule_name,
usize::try_from(action_index).ok(),
None,
) {
return override_.dispose == CoordinateDispose::Hook;
}
if sem_unknown != SemUnknownPolicy::Hook {
return false;
}
!matches!(
actions
.iter()
.find(|(candidate, _)| candidate == coordinate)
.map(|(_, template)| template),
Some(ActionTemplate::LexerPopMode)
)
})
}
/// Compiles the lexer DFA at generation time and flattens it for embedding.
///
/// An empty stream makes the generated lexer fall back to compiling the DFA
/// from its ATN at first use, so generation never fails on this step.
fn compiled_lexer_dfa_words(data: &CodegenData<'_>) -> String {
if !data.lexer_dfa_words.is_empty() {
return data
.lexer_dfa_words
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(",");
}
if data.lexer_atn.is_none() && data.lexer_atn_words.is_empty() {
return String::new();
}
let Ok(atn) = data.lexer_atn() else {
return String::new();
};
let words = CompiledLexerDfa::compile(atn).serialize();
let rendered: Vec<String> = words.iter().map(u32::to_string).collect();
rendered.join(",")
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct GeneratedParserRule {
rule_index: usize,
entry_state: usize,
left_recursive: bool,
steps: Vec<GeneratedParserStep>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum GeneratedParserStep {
MatchToken {
token_type: i32,
follow_state: usize,
},
MatchSet {
token_set: Option<usize>,
intervals: Vec<(i32, i32)>,
follow_state: usize,
},
MatchNotSet {
token_set: Option<usize>,
intervals: Vec<(i32, i32)>,
follow_state: usize,
},
MatchWildcard {
follow_state: usize,
},
Precedence(i32),
Predicate {
rule_index: usize,
pred_index: usize,
},
Action {
source_state: usize,
rule_index: usize,
},
CallRule {
source_state: usize,
rule_index: usize,
precedence: GeneratedRuleCallPrecedence,
},
Decision {
state: usize,
decision: usize,
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
fast_path: Option<GeneratedDecisionFastPath>,
alts: Vec<Vec<Self>>,
},
StarLoop {
state: usize,
decision: usize,
enter_alt: usize,
exit_alt: usize,
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
/// `true` for a `+` (one-or-more) loop, `false` for a `*` (zero-or-more)
/// loop. A `+` loop's mandatory first element is iteration 1, so its first
/// loop-back sync recovers like ANTLR's `STAR_LOOP_BACK`/`PLUS_LOOP_BACK`
/// (multi-token `consumeUntil`); a `*` loop's first sync is at the entry,
/// which recovers like `STAR_LOOP_ENTRY` (single-token deletion).
plus_loop: bool,
fast_path: Option<GeneratedDecisionFastPath>,
body: Vec<Self>,
},
LeftRecursiveLoop {
state: usize,
decision: usize,
enter_alt: usize,
exit_alt: usize,
rule_index: usize,
entry_state: usize,
body: Vec<Self>,
},
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GeneratedRuleCallPrecedence {
Literal(i32),
InheritLocal,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct GeneratedDecisionFastPath {
arms: Vec<GeneratedDecisionFastArm>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct GeneratedDecisionFastArm {
alt: usize,
intervals: Vec<(i32, i32)>,
}
#[derive(Clone, Copy)]
struct DecisionRender<'a> {
state: usize,
decision: usize,
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
fast_path: Option<&'a GeneratedDecisionFastPath>,
alts: &'a [Vec<GeneratedParserStep>],
}
#[derive(Clone, Copy)]
struct StarLoopRender<'a> {
state: usize,
decision: usize,
alts: (usize, usize),
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
plus_loop: bool,
fast_path: Option<&'a GeneratedDecisionFastPath>,
body: &'a [GeneratedParserStep],
}
#[derive(Clone, Copy)]
struct LeftRecursiveLoopRender<'a> {
state: usize,
decision: usize,
alts: (usize, usize),
rule: (usize, usize),
body: &'a [GeneratedParserStep],
}
/// Embedded-mode data consulted while rendering rule bodies: verbatim
/// action/predicate expressions plus per-rule `@init` / `@after` bodies.
#[derive(Clone, Copy)]
struct EmbeddedStepRender<'a> {
/// Keep every decision on the adaptive simulator (no LL(1)/fast-path
/// shortcuts) regardless of the tool classification.
force_adaptive: bool,
/// Tool-classified non-LL(1) decisions ([`tool_decision_analysis`]).
adaptive_decisions: &'a BTreeSet<usize>,
/// Tool LOOK(1) dispatch intervals for LL(1)-disjoint decisions.
ll1_decision_arms: &'a BTreeMap<usize, Vec<Vec<(i32, i32)>>>,
predicates: &'a BTreeMap<(usize, usize), (String, Option<String>)>,
rule_has_attrs: &'a [bool],
init_entry: &'a BTreeMap<usize, String>,
after: &'a BTreeMap<usize, String>,
call_args: &'a BTreeMap<usize, String>,
rule_arg0: &'a [Option<String>],
}
impl EmbeddedStepRender<'_> {
/// Java routes this decision through `adaptivePredict` — only there are
/// DFA states learned and full-context diagnostics emitted — so the
/// generated code must skip its LL(1)/fast-path shortcuts for it.
fn adaptive_decision(&self, decision: usize) -> bool {
self.force_adaptive || self.adaptive_decisions.contains(&decision)
}
/// Complete dispatch table for a tool-LL(1) decision, exit alternatives
/// included — Java's switch compilation. Legit input never reaches the
/// simulator through this, so no DFA state is ever learned for the
/// decision (matching the dump).
fn tool_ll1_fast_path(&self, decision: usize) -> Option<GeneratedDecisionFastPath> {
let arms = self.ll1_decision_arms.get(&decision)?;
Some(GeneratedDecisionFastPath {
arms: arms
.iter()
.enumerate()
.map(|(index, intervals)| GeneratedDecisionFastArm {
alt: index + 1,
intervals: intervals.clone(),
})
.collect(),
})
}
}
#[derive(Clone, Copy)]
struct PortableLocalStepRender<'a> {
declarations: &'a [Vec<String>],
inline_actions: &'a BTreeMap<usize, String>,
predicates: &'a BTreeMap<(usize, usize), (String, Option<String>)>,
required_generated_rules: &'a BTreeSet<usize>,
}
#[derive(Clone, Copy)]
struct GeneratedStepRenderContext<'a> {
/// `Some` in embedded mode: actions/predicates are verbatim Rust.
embedded: Option<EmbeddedStepRender<'a>>,
/// Portable raw-grammar boolean locals translated without embedded mode.
portable_locals: Option<PortableLocalStepRender<'a>>,
inline_action_statements: &'a BTreeMap<usize, String>,
track_alt_numbers: bool,
track_context_alt_numbers: bool,
direct_generated_rule_calls: &'a [bool],
atn_preferred_rule_calls: &'a [bool],
}
struct GeneratedParserCompileContext<'a> {
atn: &'a ParserAtn,
decision_by_state: &'a [Option<usize>],
rule_args: &'a [(usize, usize, RuleArgTemplate)],
inline_action_states: &'a BTreeSet<usize>,
action_states: &'a BTreeSet<usize>,
generated_action_states: &'a BTreeSet<usize>,
predicate_coordinates: &'a BTreeSet<(usize, usize)>,
generated_predicate_coordinates: &'a BTreeSet<(usize, usize)>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct TypedHookMapping {
rule_index: usize,
pred_index: usize,
method_name: String,
call: SemanticHelperCall,
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
enum LexerTypedHookKind {
Predicate,
Action,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct LexerTypedHookMapping {
rule_index: usize,
coordinate_index: usize,
kind: LexerTypedHookKind,
method_name: String,
call: SemanticHelperCall,
}
#[derive(Clone, Copy, Debug)]
struct ParserRenderOptions<'a> {
require_generated_parser: bool,
/// Splice verbatim Rust action/predicate bodies from the grammar
/// (`--actions embedded`).
embedded: bool,
generate_listener: bool,
generate_visitor: bool,
sem_unknown: SemUnknownPolicy,
patterns: Option<&'a SemPatternFile>,
}
impl Default for ParserRenderOptions<'_> {
fn default() -> Self {
Self {
require_generated_parser: false,
embedded: false,
generate_listener: true,
generate_visitor: false,
sem_unknown: SemUnknownPolicy::default(),
patterns: None,
}
}
}
#[derive(Clone, Copy)]
struct ActionStateSets<'a> {
all: &'a BTreeSet<usize>,
generated: &'a BTreeSet<usize>,
inline: &'a BTreeSet<usize>,
}
#[derive(Clone, Copy)]
struct PredicateCoordinateSets<'a> {
all: &'a BTreeSet<(usize, usize)>,
generated: &'a BTreeSet<(usize, usize)>,
}
const fn generated_action_state_sets<'a>(
context: &GeneratedParserCompileContext<'a>,
) -> ActionStateSets<'a> {
ActionStateSets {
all: context.action_states,
generated: context.generated_action_states,
inline: context.inline_action_states,
}
}
const fn generated_predicate_coordinate_sets<'a>(
context: &GeneratedParserCompileContext<'a>,
) -> PredicateCoordinateSets<'a> {
PredicateCoordinateSets {
all: context.predicate_coordinates,
generated: context.generated_predicate_coordinates,
}
}
/// Compiles the parser ATN subset that is safe to emit as recursive-descent
/// Rust today. Unsupported states deliberately return `None` so the generated
/// method can keep using the interpreter fallback until more ATN shapes are
/// covered.
fn parser_generated_rules(
data: &CodegenData<'_>,
enabled_rules: &[bool],
rule_args: &[(usize, usize, RuleArgTemplate)],
action_states: ActionStateSets<'_>,
predicate_coordinates: PredicateCoordinateSets<'_>,
require_generated_callees: bool,
) -> io::Result<Vec<Option<GeneratedParserRule>>> {
let atn = data.parser_atn()?;
let decision_by_state = decision_by_state(atn);
let context = GeneratedParserCompileContext {
atn,
decision_by_state: &decision_by_state,
rule_args,
inline_action_states: action_states.inline,
action_states: action_states.all,
generated_action_states: action_states.generated,
predicate_coordinates: predicate_coordinates.all,
generated_predicate_coordinates: predicate_coordinates.generated,
};
let mut rules = (0..data.rule_names.len())
.map(|rule_index| {
if enabled_rules.get(rule_index).copied().unwrap_or_default() {
compile_generated_parser_rule(&context, rule_index)
} else {
None
}
})
.collect::<Vec<_>>();
if require_generated_callees {
drop_rules_calling_disabled_rules(&mut rules);
}
Ok(rules)
}
fn drop_rules_calling_disabled_rules(rules: &mut [Option<GeneratedParserRule>]) {
loop {
let enabled = rules.iter().map(Option::is_some).collect::<Vec<_>>();
let drop_index = rules.iter().filter_map(Option::as_ref).find_map(|rule| {
generated_steps_call_disabled_rule(&rule.steps, &enabled).then_some(rule.rule_index)
});
let Some(rule_index) = drop_index else {
return;
};
rules[rule_index] = None;
}
}
const ATN_PREFERRED_LEADING_CALL_CHAIN_MIN: usize = 8;
const ATN_PREFERRED_CHAIN_MIN_DECISION_DENSITY_NUMERATOR: usize = 2;
const ATN_PREFERRED_WRAPPER_MIN_DECISION_COST: usize = 2;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct GeneratedRuleShape {
decision_cost: usize,
action_or_predicate_count: usize,
}
impl AddAssign for GeneratedRuleShape {
fn add_assign(&mut self, rhs: Self) {
self.decision_cost += rhs.decision_cost;
self.action_or_predicate_count += rhs.action_or_predicate_count;
}
}
#[cfg(test)]
fn generated_atn_preferred_rule_calls(
rules: &[Option<GeneratedParserRule>],
rule_names: &[String],
) -> Vec<bool> {
generated_atn_preferred_rule_calls_excluding(rules, rule_names, &BTreeSet::new())
}
fn generated_atn_preferred_rule_calls_excluding(
rules: &[Option<GeneratedParserRule>],
_rule_names: &[String],
force_generated: &BTreeSet<usize>,
) -> Vec<bool> {
let leading_rule_calls = rules
.iter()
.map(|rule| {
rule.as_ref()
.and_then(|rule| generated_steps_leading_mandatory_rule_call(&rule.steps))
})
.collect::<Vec<_>>();
let shapes = rules
.iter()
.map(|rule| {
rule.as_ref()
.map_or_else(GeneratedRuleShape::default, generated_rule_shape)
})
.collect::<Vec<_>>();
let mut preferred = vec![false; rules.len()];
for start in 0..rules.len() {
if rules[start].is_none() {
continue;
}
let mut chain = Vec::new();
let mut seen = vec![false; rules.len()];
let mut current = start;
loop {
if current >= rules.len() || rules[current].is_none() || seen[current] {
break;
}
seen[current] = true;
chain.push(current);
let Some(next) = leading_rule_calls[current] else {
break;
};
current = next;
}
if chain.len() >= ATN_PREFERRED_LEADING_CALL_CHAIN_MIN
&& generated_atn_preferred_chain_is_expensive(&chain, &shapes)
{
for rule_index in chain {
preferred[rule_index] = true;
}
}
}
propagate_atn_preferred_wrappers(rules, &shapes, &mut preferred);
for rule_index in force_generated {
if let Some(entry) = preferred.get_mut(*rule_index) {
*entry = false;
}
}
preferred
}
fn generated_rule_callers_reaching(
rules: &[Option<GeneratedParserRule>],
target_rules: &BTreeSet<usize>,
) -> BTreeSet<usize> {
let mut graph = DiGraph::new();
let nodes = (0..rules.len())
.map(|rule_index| graph.add_node(rule_index))
.collect::<Vec<_>>();
for rule in rules.iter().flatten() {
let mut callees = BTreeSet::new();
collect_generated_step_callees(&rule.steps, &mut callees);
for callee in callees {
if let Some(target) = nodes.get(callee) {
graph.add_edge(nodes[rule.rule_index], *target, ());
}
}
}
graph_nodes_reaching(&graph, target_rules)
}
fn parser_rule_callers_reaching(
data: &CodegenData<'_>,
target_rules: &BTreeSet<usize>,
) -> io::Result<BTreeSet<usize>> {
if target_rules.is_empty() {
return Ok(BTreeSet::new());
}
if let Some(semantic) = data.semantic {
let mut graph = DiGraph::new();
let nodes = (0..data.rule_names.len())
.map(|rule_index| graph.add_node(rule_index))
.collect::<Vec<_>>();
for (caller, callees) in &semantic.call_graph {
let caller = semantic.recognizer.rule_numbers[caller];
for callee in callees {
graph.add_edge(
nodes[caller],
nodes[semantic.recognizer.rule_numbers[callee]],
(),
);
}
}
return Ok(graph_nodes_reaching(&graph, target_rules));
}
let atn = data.parser_atn()?;
Ok(atn_rule_callers_reaching(
atn,
target_rules,
data.rule_names.len(),
))
}
fn atn_rule_callers_reaching(
atn: &ParserAtn,
target_rules: &BTreeSet<usize>,
rule_count: usize,
) -> BTreeSet<usize> {
let mut reaching = target_rules.clone();
loop {
let mut changed = false;
for state in atn.states() {
let Some(caller_rule) = state.rule_index().filter(|index| *index < rule_count) else {
continue;
};
if reaching.contains(&caller_rule) {
continue;
}
let calls_reaching_rule = state.transitions().iter().any(|transition| {
matches!(
transition.data(),
ParserTransitionData::Rule { rule_index, .. }
if reaching.contains(&rule_index)
)
});
if calls_reaching_rule {
changed |= reaching.insert(caller_rule);
}
}
if !changed {
return reaching;
}
}
}
fn collect_generated_step_callees(steps: &[GeneratedParserStep], callees: &mut BTreeSet<usize>) {
for step in steps {
match step {
GeneratedParserStep::CallRule { rule_index, .. } => {
callees.insert(*rule_index);
}
GeneratedParserStep::Decision { alts, .. } => {
for alternative in alts {
collect_generated_step_callees(alternative, callees);
}
}
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
collect_generated_step_callees(body, callees);
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => {}
}
}
}
fn graph_nodes_reaching(graph: &DiGraph<usize, ()>, targets: &BTreeSet<usize>) -> BTreeSet<usize> {
let reversed = Reversed(graph);
let mut traversal = Dfs::empty(reversed);
let mut reaching = targets.clone();
for target in graph
.node_indices()
.filter(|node| targets.contains(&graph[*node]))
{
traversal.move_to(target);
while let Some(node) = traversal.next(reversed) {
reaching.insert(graph[node]);
}
}
reaching
}
fn generated_atn_preferred_chain_is_expensive(
chain: &[usize],
shapes: &[GeneratedRuleShape],
) -> bool {
let decision_cost = chain
.iter()
.filter_map(|rule_index| shapes.get(*rule_index))
.map(|shape| shape.decision_cost)
.sum::<usize>();
decision_cost >= chain.len() * ATN_PREFERRED_CHAIN_MIN_DECISION_DENSITY_NUMERATOR
}
fn propagate_atn_preferred_wrappers(
rules: &[Option<GeneratedParserRule>],
shapes: &[GeneratedRuleShape],
preferred: &mut [bool],
) {
loop {
let mut changed = false;
for (rule_index, rule) in rules.iter().enumerate() {
if preferred.get(rule_index).copied().unwrap_or_default() {
continue;
}
let Some(rule) = rule else {
continue;
};
if !generated_rule_is_atn_preferred_wrapper(rule, shapes, preferred) {
continue;
}
preferred[rule_index] = true;
changed = true;
}
if !changed {
return;
}
}
}
fn generated_rule_is_atn_preferred_wrapper(
rule: &GeneratedParserRule,
shapes: &[GeneratedRuleShape],
preferred: &[bool],
) -> bool {
if rule.left_recursive {
return false;
}
let shape = shapes.get(rule.rule_index).copied().unwrap_or_default();
shape.action_or_predicate_count == 0
&& shape.decision_cost >= ATN_PREFERRED_WRAPPER_MIN_DECISION_COST
&& generated_steps_call_atn_preferred_rule(&rule.steps, preferred)
}
fn generated_rule_shape(rule: &GeneratedParserRule) -> GeneratedRuleShape {
generated_steps_shape(&rule.steps)
}
fn generated_steps_shape(steps: &[GeneratedParserStep]) -> GeneratedRuleShape {
let mut shape = GeneratedRuleShape::default();
for step in steps {
shape += generated_step_shape(step);
}
shape
}
fn generated_step_shape(step: &GeneratedParserStep) -> GeneratedRuleShape {
match step {
GeneratedParserStep::Decision {
allow_semantic_context,
force_context,
fast_path,
alts,
..
} => {
let mut shape = GeneratedRuleShape {
decision_cost: usize::from(
fast_path.is_none() || *allow_semantic_context || *force_context,
),
action_or_predicate_count: 0,
};
for alt in alts {
shape += generated_steps_shape(alt);
}
shape
}
GeneratedParserStep::StarLoop {
allow_semantic_context,
force_context,
fast_path,
body,
..
} => {
let mut shape = GeneratedRuleShape {
decision_cost: usize::from(
fast_path.is_none() || *allow_semantic_context || *force_context,
),
action_or_predicate_count: 0,
};
shape += generated_steps_shape(body);
shape
}
GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
let mut shape = GeneratedRuleShape {
decision_cost: 1,
action_or_predicate_count: 0,
};
shape += generated_steps_shape(body);
shape
}
GeneratedParserStep::Predicate { .. } | GeneratedParserStep::Action { .. } => {
GeneratedRuleShape {
decision_cost: 0,
action_or_predicate_count: 1,
}
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::CallRule { .. } => GeneratedRuleShape::default(),
}
}
fn generated_steps_call_atn_preferred_rule(
steps: &[GeneratedParserStep],
preferred: &[bool],
) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::CallRule { rule_index, .. } => {
preferred.get(*rule_index).copied().unwrap_or_default()
}
GeneratedParserStep::Decision { alts, .. } => alts
.iter()
.any(|alt| generated_steps_call_atn_preferred_rule(alt, preferred)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
generated_steps_call_atn_preferred_rule(body, preferred)
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => false,
})
}
fn generated_steps_leading_mandatory_rule_call(steps: &[GeneratedParserStep]) -> Option<usize> {
for step in steps {
match step {
GeneratedParserStep::CallRule { rule_index, .. } => return Some(*rule_index),
GeneratedParserStep::Decision { alts, .. } if generated_alts_are_nullable(alts) => {}
GeneratedParserStep::Decision { alts, .. } => {
return generated_alts_common_leading_mandatory_rule_call(alts);
}
GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => {}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. } => return None,
}
}
None
}
fn generated_alts_common_leading_mandatory_rule_call(
alts: &[Vec<GeneratedParserStep>],
) -> Option<usize> {
let mut common = None;
for alt in alts {
let rule_index = generated_steps_leading_mandatory_rule_call(alt)?;
match common {
Some(common_rule_index) if common_rule_index != rule_index => return None,
Some(_) => {}
None => common = Some(rule_index),
}
}
common
}
fn generated_alts_are_nullable(alts: &[Vec<GeneratedParserStep>]) -> bool {
alts.iter().any(|alt| generated_steps_are_nullable(alt))
}
fn generated_steps_are_nullable(steps: &[GeneratedParserStep]) -> bool {
steps.iter().all(generated_step_is_nullable)
}
fn generated_step_is_nullable(step: &GeneratedParserStep) -> bool {
match step {
GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => true,
GeneratedParserStep::Decision { alts, .. } => generated_alts_are_nullable(alts),
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. } => false,
}
}
fn require_all_parser_rules_generated(
rules: &[Option<GeneratedParserRule>],
data: &CodegenData<'_>,
) -> io::Result<()> {
let missing = rules
.iter()
.enumerate()
.filter(|(_, rule)| rule.is_none())
.map(|(index, _)| {
data.rule_names
.get(index)
.map_or_else(|| index.to_string(), Clone::clone)
})
.collect::<Vec<_>>();
if missing.is_empty() {
return Ok(());
}
Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"generated parser did not emit {} rule(s): {}",
missing.len(),
missing.join(", ")
),
))
}
fn require_portable_local_rules_generated(
rules: &[Option<GeneratedParserRule>],
required: &BTreeSet<usize>,
data: &CodegenData<'_>,
) -> io::Result<()> {
let missing = required
.iter()
.filter(|rule_index| rules.get(**rule_index).is_none_or(Option::is_none))
.map(|rule_index| {
data.rule_names
.get(*rule_index)
.map_or_else(|| rule_index.to_string(), Clone::clone)
})
.collect::<Vec<_>>();
if missing.is_empty() {
return Ok(());
}
Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"portable local semantics require {} generated parser rule(s): {}",
missing.len(),
missing.join(", ")
),
))
}
fn generated_steps_call_disabled_rule(steps: &[GeneratedParserStep], enabled: &[bool]) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::CallRule { rule_index, .. } => {
!enabled.get(*rule_index).copied().unwrap_or_default()
}
GeneratedParserStep::Decision { alts, .. } => alts
.iter()
.any(|alt| generated_steps_call_disabled_rule(alt, enabled)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
generated_steps_call_disabled_rule(body, enabled)
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => false,
})
}
fn decision_by_state(atn: &ParserAtn) -> Vec<Option<usize>> {
let mut decision_by_state = vec![None; atn.state_count()];
for (decision, state_number) in atn.decision_to_state().iter().enumerate() {
if let Some(slot) = decision_by_state.get_mut(state_number) {
*slot = Some(decision);
}
}
decision_by_state
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
struct GeneratedLookSet {
symbols: BTreeSet<i32>,
nullable: bool,
}
#[derive(Default)]
struct GeneratedFirstSetCtx {
cache: BTreeMap<(usize, usize), GeneratedLookSet>,
in_progress: BTreeSet<(usize, usize)>,
hit_cycle: bool,
}
fn generated_decision_fast_path<'a>(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
alts: impl IntoIterator<Item = (usize, &'a [GeneratedParserStep])>,
) -> Option<GeneratedDecisionFastPath> {
if state.precedence_rule_decision() || state.non_greedy() {
return None;
}
let mut first_ctx = GeneratedFirstSetCtx::default();
let mut symbol_alts = BTreeMap::<i32, Option<usize>>::new();
for (alt, steps) in alts {
let look = generated_steps_first_set(context.atn, steps, &mut first_ctx);
if look.nullable {
return None;
}
for symbol in look.symbols {
match symbol_alts.get(&symbol).copied().flatten() {
None if symbol_alts.contains_key(&symbol) => {}
None => {
symbol_alts.insert(symbol, Some(alt));
}
Some(existing) if existing == alt => {}
Some(_) => {
symbol_alts.insert(symbol, None);
}
}
}
}
let mut symbols_by_alt = BTreeMap::<usize, BTreeSet<i32>>::new();
for (symbol, alt) in symbol_alts {
if let Some(alt) = alt {
symbols_by_alt.entry(alt).or_default().insert(symbol);
}
}
let arms = symbols_by_alt
.into_iter()
.map(|(alt, symbols)| GeneratedDecisionFastArm {
alt,
intervals: symbols_to_ranges(symbols),
})
.filter(|arm| !arm.intervals.is_empty())
.collect::<Vec<_>>();
(!arms.is_empty()).then_some(GeneratedDecisionFastPath { arms })
}
fn generated_steps_first_set(
atn: &ParserAtn,
steps: &[GeneratedParserStep],
ctx: &mut GeneratedFirstSetCtx,
) -> GeneratedLookSet {
let mut first = GeneratedLookSet::default();
for step in steps {
match step {
GeneratedParserStep::MatchToken { token_type, .. } => {
first.symbols.insert(*token_type);
first.nullable = false;
return first;
}
GeneratedParserStep::MatchSet { intervals, .. } => {
for (start, stop) in intervals {
first.symbols.extend(*start..=*stop);
}
first.nullable = false;
return first;
}
GeneratedParserStep::MatchNotSet { intervals, .. } => {
first.symbols.extend(1..=atn.max_token_type());
for (start, stop) in intervals {
for symbol in *start..=*stop {
first.symbols.remove(&symbol);
}
}
first.nullable = false;
return first;
}
GeneratedParserStep::MatchWildcard { .. } => {
first.symbols.extend(1..=atn.max_token_type());
first.nullable = false;
return first;
}
GeneratedParserStep::CallRule { rule_index, .. } => {
let Some(start) = atn.rule_to_start_state().get(*rule_index) else {
return GeneratedLookSet::default();
};
let Some(stop) = atn.rule_to_stop_state().get(*rule_index) else {
return GeneratedLookSet::default();
};
let child = generated_rule_first_set(atn, start, stop, ctx);
first.symbols.extend(child.symbols);
if !child.nullable {
first.nullable = false;
return first;
}
}
GeneratedParserStep::Decision { alts, .. } => {
let nested = generated_alt_steps_first_set(atn, alts, ctx);
first.symbols.extend(nested.symbols);
if !nested.nullable {
first.nullable = false;
return first;
}
}
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
let nested = generated_steps_first_set(atn, body, ctx);
first.symbols.extend(nested.symbols);
}
GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => {}
}
}
first.nullable = true;
first
}
fn generated_alt_steps_first_set(
atn: &ParserAtn,
alts: &[Vec<GeneratedParserStep>],
ctx: &mut GeneratedFirstSetCtx,
) -> GeneratedLookSet {
let mut first = GeneratedLookSet::default();
for alt in alts {
let alt_first = generated_steps_first_set(atn, alt, ctx);
first.symbols.extend(alt_first.symbols);
first.nullable |= alt_first.nullable;
}
first
}
fn generated_rule_first_set(
atn: &ParserAtn,
state_number: usize,
rule_stop_state: usize,
ctx: &mut GeneratedFirstSetCtx,
) -> GeneratedLookSet {
let key = (state_number, rule_stop_state);
if let Some(cached) = ctx.cache.get(&key) {
return cached.clone();
}
if !ctx.in_progress.insert(key) {
return GeneratedLookSet::default();
}
let saved_hit_cycle = ctx.hit_cycle;
ctx.hit_cycle = false;
let mut first = GeneratedLookSet::default();
generated_rule_first_set_inner(
atn,
state_number,
rule_stop_state,
ctx,
&mut BTreeSet::new(),
&mut first,
);
ctx.in_progress.remove(&key);
if !ctx.hit_cycle {
ctx.cache.insert(key, first.clone());
}
ctx.hit_cycle = saved_hit_cycle || ctx.hit_cycle;
first
}
fn generated_rule_first_set_inner(
atn: &ParserAtn,
state_number: usize,
rule_stop_state: usize,
ctx: &mut GeneratedFirstSetCtx,
visited: &mut BTreeSet<usize>,
first: &mut GeneratedLookSet,
) {
if !visited.insert(state_number) {
return;
}
if state_number == rule_stop_state {
first.nullable = true;
return;
}
let Some(state) = atn.state(state_number) else {
return;
};
for transition in state.transitions() {
let symbols = generated_transition_symbols(transition, atn.max_token_type());
if !symbols.is_empty() {
first.symbols.extend(symbols);
continue;
}
match transition.data() {
ParserTransitionData::Epsilon { target }
| ParserTransitionData::Action { target, .. }
| ParserTransitionData::Predicate { target, .. }
| ParserTransitionData::Precedence { target, .. } => {
generated_rule_first_set_inner(atn, target, rule_stop_state, ctx, visited, first);
}
ParserTransitionData::Rule {
target,
rule_index,
follow_state,
..
} => {
let Some(child_stop) = atn.rule_to_stop_state().get(rule_index) else {
continue;
};
let child_key = (target, child_stop);
if ctx.in_progress.contains(&child_key) && !ctx.cache.contains_key(&child_key) {
ctx.hit_cycle = true;
}
let child = generated_rule_first_set(atn, target, child_stop, ctx);
first.symbols.extend(child.symbols);
if child.nullable {
generated_rule_first_set_inner(
atn,
follow_state,
rule_stop_state,
ctx,
visited,
first,
);
}
}
ParserTransitionData::Atom { .. }
| ParserTransitionData::Range { .. }
| ParserTransitionData::Set { .. }
| ParserTransitionData::NotSet { .. }
| ParserTransitionData::Wildcard { .. } => {}
}
}
}
fn generated_transition_symbols(
transition: ParserTransition<'_>,
max_token_type: i32,
) -> BTreeSet<i32> {
let mut symbols = BTreeSet::new();
match transition.data() {
ParserTransitionData::Atom { label, .. } => {
symbols.insert(label);
}
ParserTransitionData::Range { start, stop, .. } => {
symbols.extend(start..=stop);
}
ParserTransitionData::Set { set, .. } => {
for (start, stop) in set.ranges() {
symbols.extend(start..=stop);
}
}
ParserTransitionData::NotSet { set, .. } => {
symbols.extend((1..=max_token_type).filter(|symbol| !set.contains(*symbol)));
}
ParserTransitionData::Wildcard { .. } => {
symbols.extend(1..=max_token_type);
}
ParserTransitionData::Epsilon { .. }
| ParserTransitionData::Rule { .. }
| ParserTransitionData::Predicate { .. }
| ParserTransitionData::Action { .. }
| ParserTransitionData::Precedence { .. } => {}
}
symbols
}
fn symbols_to_ranges(symbols: BTreeSet<i32>) -> Vec<(i32, i32)> {
let mut ranges = Vec::new();
for symbol in symbols {
match ranges.last_mut() {
Some((_, stop)) if *stop + 1 == symbol => *stop = symbol,
_ => ranges.push((symbol, symbol)),
}
}
ranges
}
fn state_tracks_alt_number(state: ParserAtnState<'_>) -> bool {
matches!(
state.kind(),
AtnStateKind::Basic
| AtnStateKind::BlockStart
| AtnStateKind::PlusBlockStart
| AtnStateKind::StarBlockStart
| AtnStateKind::StarLoopEntry
) && !state.precedence_rule_decision()
&& state.transitions().len() > 1
}
fn compile_generated_parser_rule(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
) -> Option<GeneratedParserRule> {
let entry_state = context.atn.rule_to_start_state().get(rule_index)?;
let stop_state = context.atn.rule_to_stop_state().get(rule_index)?;
let start = context.atn.state(entry_state)?;
if start.left_recursive_rule() {
return compile_generated_left_recursive_parser_rule(
context,
rule_index,
entry_state,
stop_state,
);
}
let mut visited = BTreeSet::new();
let steps = compile_generated_parser_path(context, entry_state, stop_state, &mut visited)?;
Some(GeneratedParserRule {
rule_index,
entry_state,
left_recursive: false,
steps,
})
}
fn compile_generated_left_recursive_parser_rule(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
entry_state: usize,
stop_state: usize,
) -> Option<GeneratedParserRule> {
let loop_entry = find_left_recursive_loop_entry(context, rule_index)?;
let mut visited = BTreeSet::new();
let mut steps = compile_generated_parser_path(context, entry_state, loop_entry, &mut visited)?;
let loop_state = context.atn.state(loop_entry)?;
let decision = context
.decision_by_state
.get(loop_entry)
.copied()
.flatten()?;
let (loop_step, exit_target) = compile_generated_left_recursive_loop(
context,
rule_index,
entry_state,
loop_state,
decision,
)?;
steps.push(loop_step);
steps.extend(compile_generated_parser_path(
context,
exit_target,
stop_state,
&mut BTreeSet::new(),
)?);
Some(GeneratedParserRule {
rule_index,
entry_state,
left_recursive: true,
steps,
})
}
fn find_left_recursive_loop_entry(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
) -> Option<usize> {
context.atn.states().find_map(|state| {
(state.rule_index() == Some(rule_index)
&& state.kind() == AtnStateKind::StarLoopEntry
&& state.precedence_rule_decision())
.then_some(state.state_number())
})
}
fn compile_generated_left_recursive_loop(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
entry_state: usize,
state: ParserAtnState<'_>,
decision: usize,
) -> Option<(GeneratedParserStep, usize)> {
let mut enter = None;
let mut exit = None;
for (index, transition) in state.transitions().iter().enumerate() {
let alt = index + 1;
let target = transition.target();
let target_state = context.atn.state(target)?;
if target_state.kind() == AtnStateKind::LoopEnd {
exit = Some((alt, transition, target, target_state.loop_back_state()?));
} else {
enter = Some((alt, transition));
}
}
let (enter_alt, enter_transition) = enter?;
let (exit_alt, exit_transition, exit_target, loop_back_state) = exit?;
let (enter_step, enter_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
enter_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut body = enter_step.into_iter().collect::<Vec<_>>();
body.extend(compile_generated_parser_path(
context,
enter_target,
loop_back_state,
&mut BTreeSet::new(),
)?);
allow_semantic_context_in_decisions(&mut body);
if !steps_may_consume(&body) {
return None;
}
let (exit_step, _) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
exit_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
if exit_step.is_some() {
return None;
}
Some((
GeneratedParserStep::LeftRecursiveLoop {
state: state.state_number(),
decision,
enter_alt,
exit_alt,
rule_index,
entry_state,
body,
},
exit_target,
))
}
fn compile_generated_parser_path(
context: &GeneratedParserCompileContext<'_>,
state_number: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
if state_number == stop_state {
return Some(Vec::new());
}
if !visited.insert(state_number) {
return None;
}
let state = context.atn.state(state_number)?;
let steps = if let Some(decision) = context
.decision_by_state
.get(state_number)
.copied()
.flatten()
{
compile_generated_parser_decision_state(context, state, decision, stop_state, visited)?
} else {
let transition = state.transitions().first()?;
if state.transitions().len() != 1 {
return None;
}
let (step, target) = compile_generated_parser_transition(
state_number,
context.rule_args,
transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut steps = step.into_iter().collect::<Vec<_>>();
steps.extend(compile_generated_parser_path(
context, target, stop_state, visited,
)?);
steps
};
visited.remove(&state_number);
Some(steps)
}
fn compile_generated_parser_decision_state(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
match state.kind() {
AtnStateKind::BlockStart | AtnStateKind::PlusBlockStart | AtnStateKind::StarBlockStart => {
compile_generated_parser_block_decision(context, state, decision, stop_state, visited)
}
AtnStateKind::StarLoopEntry => {
compile_generated_parser_star_loop(context, state, decision, stop_state, visited)
}
AtnStateKind::PlusLoopBack => {
compile_generated_parser_plus_loop(context, state, decision, stop_state, visited)
}
_ => None,
}
}
fn compile_generated_parser_block_decision(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
let end_state = state.end_state()?;
let mut alts = Vec::with_capacity(state.transitions().len());
for transition in state.transitions() {
let (step, target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut alt_visited = visited.clone();
let mut alt_steps = step.into_iter().collect::<Vec<_>>();
alt_steps.extend(compile_generated_parser_path(
context,
target,
end_state,
&mut alt_visited,
)?);
alts.push(alt_steps);
}
let mut steps = vec![GeneratedParserStep::Decision {
state: state.state_number(),
decision,
track_alt_number: state_tracks_alt_number(state),
allow_semantic_context: alts.iter().any(|alt| steps_contain_predicate(alt)),
force_context: state.non_greedy(),
fast_path: generated_decision_fast_path(
context,
state,
alts.iter()
.enumerate()
.map(|(index, alt)| (index + 1, alt.as_slice())),
),
alts,
}];
steps.extend(compile_generated_parser_path(
context, end_state, stop_state, visited,
)?);
Some(steps)
}
fn compile_generated_parser_star_loop(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
let mut enter = None;
let mut exit = None;
for (index, transition) in state.transitions().iter().enumerate() {
let alt = index + 1;
let target = transition.target();
let target_state = context.atn.state(target)?;
let target_kind = target_state.kind();
if target_kind == AtnStateKind::LoopEnd {
exit = Some((alt, transition, target_state.loop_back_state()?));
} else {
enter = Some((alt, transition));
}
}
let (enter_alt, enter_transition) = enter?;
let (exit_alt, exit_transition, loop_back_state) = exit?;
let (enter_step, enter_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
enter_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut body_visited = BTreeSet::new();
let mut body = enter_step.into_iter().collect::<Vec<_>>();
body.extend(compile_generated_parser_path(
context,
enter_target,
loop_back_state,
&mut body_visited,
)?);
if !steps_may_consume(&body) {
return None;
}
let (exit_step, exit_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
exit_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
if exit_step.is_some() {
return None;
}
let mut steps = vec![GeneratedParserStep::StarLoop {
state: state.state_number(),
decision,
enter_alt,
exit_alt,
track_alt_number: state_tracks_alt_number(state),
allow_semantic_context: steps_contain_predicate(&body),
force_context: state.non_greedy(),
plus_loop: false,
fast_path: None,
body,
}];
steps.extend(compile_generated_parser_path(
context,
exit_target,
stop_state,
visited,
)?);
Some(steps)
}
fn compile_generated_parser_plus_loop(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
let mut enter = None;
let mut exit = None;
for (index, transition) in state.transitions().iter().enumerate() {
let alt = index + 1;
let target = transition.target();
let target_state = context.atn.state(target)?;
if target_state.kind() == AtnStateKind::LoopEnd {
exit = Some((alt, transition));
} else {
enter = Some((alt, transition));
}
}
let (enter_alt, enter_transition) = enter?;
let (enter_step, enter_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
enter_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut body_visited = BTreeSet::new();
let mut body = enter_step.into_iter().collect::<Vec<_>>();
body.extend(compile_generated_parser_path(
context,
enter_target,
state.state_number(),
&mut body_visited,
)?);
if !steps_may_consume(&body) {
return None;
}
let (exit_alt, exit_transition) = exit?;
let (exit_step, exit_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
exit_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
if exit_step.is_some() {
return None;
}
let mut steps = vec![GeneratedParserStep::StarLoop {
state: state.state_number(),
decision,
enter_alt,
exit_alt,
track_alt_number: state_tracks_alt_number(state),
allow_semantic_context: steps_contain_predicate(&body),
force_context: state.non_greedy(),
plus_loop: true,
fast_path: None,
body,
}];
steps.extend(compile_generated_parser_path(
context,
exit_target,
stop_state,
visited,
)?);
Some(steps)
}
fn steps_may_consume(steps: &[GeneratedParserStep]) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. } => true,
GeneratedParserStep::Action { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. } => false,
GeneratedParserStep::Decision { alts, .. } => alts.iter().any(|alt| steps_may_consume(alt)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => steps_may_consume(body),
})
}
fn allow_semantic_context_in_decisions(steps: &mut [GeneratedParserStep]) {
for step in steps {
match step {
GeneratedParserStep::Decision {
allow_semantic_context,
fast_path,
alts,
..
} => {
*allow_semantic_context = true;
*fast_path = None;
for alt in alts {
allow_semantic_context_in_decisions(alt);
}
}
GeneratedParserStep::StarLoop {
allow_semantic_context,
fast_path,
body,
..
} => {
*allow_semantic_context = true;
*fast_path = None;
allow_semantic_context_in_decisions(body);
}
GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
allow_semantic_context_in_decisions(body);
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::CallRule { .. } => {}
}
}
}
fn steps_contain_predicate(steps: &[GeneratedParserStep]) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::Predicate { .. } => true,
GeneratedParserStep::Decision { alts, .. } => {
alts.iter().any(|alt| steps_contain_predicate(alt))
}
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => steps_contain_predicate(body),
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::CallRule { .. } => false,
})
}
fn generated_rule_call_precedence(
rule_args: &[(usize, usize, RuleArgTemplate)],
source_state: usize,
rule_index: usize,
transition_precedence: i32,
) -> Option<GeneratedRuleCallPrecedence> {
let Some((_, _, arg)) = rule_args
.iter()
.find(|(arg_source, arg_rule, _)| *arg_source == source_state && *arg_rule == rule_index)
else {
return Some(GeneratedRuleCallPrecedence::Literal(transition_precedence));
};
match arg {
RuleArgTemplate::Literal(value) => i32::try_from(*value)
.ok()
.map(GeneratedRuleCallPrecedence::Literal),
RuleArgTemplate::InheritLocal => Some(GeneratedRuleCallPrecedence::InheritLocal),
}
}
fn compile_generated_parser_transition(
source_state: usize,
rule_args: &[(usize, usize, RuleArgTemplate)],
transition: ParserTransition<'_>,
action_states: ActionStateSets<'_>,
predicate_coordinates: PredicateCoordinateSets<'_>,
) -> Option<(Option<GeneratedParserStep>, usize)> {
match transition.data() {
ParserTransitionData::Epsilon { target } => Some((None, target)),
ParserTransitionData::Atom { target, label } => Some((
Some(GeneratedParserStep::MatchToken {
token_type: label,
follow_state: target,
}),
target,
)),
ParserTransitionData::Range {
target,
start,
stop,
} => Some((
Some(GeneratedParserStep::MatchSet {
token_set: None,
intervals: vec![(start, stop)],
follow_state: target,
}),
target,
)),
ParserTransitionData::Set { target, set } => Some((
Some(GeneratedParserStep::MatchSet {
token_set: (set.kind() == ParserTokenSetKind::Dense).then_some(set.index()),
intervals: set.ranges().collect(),
follow_state: target,
}),
target,
)),
ParserTransitionData::NotSet { target, set } => Some((
Some(GeneratedParserStep::MatchNotSet {
token_set: (set.kind() == ParserTokenSetKind::Dense).then_some(set.index()),
intervals: set.ranges().collect(),
follow_state: target,
}),
target,
)),
ParserTransitionData::Wildcard { target } => Some((
Some(GeneratedParserStep::MatchWildcard {
follow_state: target,
}),
target,
)),
ParserTransitionData::Rule {
rule_index,
follow_state,
precedence,
..
} => Some((
Some(GeneratedParserStep::CallRule {
source_state,
rule_index,
precedence: generated_rule_call_precedence(
rule_args,
source_state,
rule_index,
precedence,
)?,
}),
follow_state,
)),
ParserTransitionData::Action {
target, rule_index, ..
} if action_states.generated.contains(&source_state) => Some((
Some(GeneratedParserStep::Action {
source_state,
rule_index,
}),
target,
)),
ParserTransitionData::Action {
target,
action_index: None,
..
} if !action_states.all.contains(&source_state) => Some((None, target)),
ParserTransitionData::Predicate {
target,
rule_index,
pred_index,
..
} if predicate_coordinates
.generated
.contains(&(rule_index, pred_index)) =>
{
Some((
Some(GeneratedParserStep::Predicate {
rule_index,
pred_index,
}),
target,
))
}
ParserTransitionData::Predicate {
rule_index,
pred_index,
..
} if predicate_coordinates
.all
.contains(&(rule_index, pred_index)) =>
{
None
}
ParserTransitionData::Predicate { target, .. } => Some((None, target)),
ParserTransitionData::Precedence { target, precedence } => {
Some((Some(GeneratedParserStep::Precedence(precedence)), target))
}
ParserTransitionData::Action { .. } => None,
}
}
#[cfg(test)]
fn render_generated_rule_dispatch(
rules: &[Option<GeneratedParserRule>],
direct_generated_rule_calls: &[bool],
inline_action_statements: &BTreeMap<usize, String>,
track_alt_numbers: bool,
) -> String {
render_generated_rule_dispatch_with_rule_names(
rules,
direct_generated_rule_calls,
&[],
inline_action_statements,
track_alt_numbers,
false,
None,
None,
)
}
#[allow(clippy::too_many_arguments)]
fn render_generated_rule_dispatch_with_rule_names(
rules: &[Option<GeneratedParserRule>],
direct_generated_rule_calls: &[bool],
rule_names: &[String],
inline_action_statements: &BTreeMap<usize, String>,
track_alt_numbers: bool,
track_context_alt_numbers: bool,
embedded: Option<EmbeddedStepRender<'_>>,
portable_locals: Option<PortableLocalStepRender<'_>>,
) -> String {
let mut out = String::new();
let force_generated_rules = if embedded.is_some() {
rules.iter().flatten().map(|rule| rule.rule_index).collect()
} else {
portable_locals.map_or_else(BTreeSet::new, |portable| {
generated_rule_callers_reaching(rules, portable.required_generated_rules)
})
};
let atn_preferred_rule_calls =
generated_atn_preferred_rule_calls_excluding(rules, rule_names, &force_generated_rules);
writeln!(
out,
" #[allow(dead_code)]\n fn parse_generated_rule(&mut self, rule_index: usize, precedence: i32, allow_fallback: bool) -> Option<Result<antlr4_runtime::ParseTree, GeneratedRuleError>> {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " let _ = precedence;").expect("writing to a string cannot fail");
writeln!(out, " let _ = allow_fallback;").expect("writing to a string cannot fail");
writeln!(out, " match rule_index {{").expect("writing to a string cannot fail");
for rule in rules.iter().flatten() {
let index = rule.rule_index;
if atn_preferred_rule_calls
.get(index)
.copied()
.unwrap_or_default()
{
writeln!(
out,
" {index} if self.generated_only() => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback)),"
)
.expect("writing to a string cannot fail");
} else {
writeln!(
out,
" {index} => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback)),"
)
.expect("writing to a string cannot fail");
}
}
writeln!(out, " _ => None,").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
let step_render_context = GeneratedStepRenderContext {
embedded,
portable_locals,
inline_action_statements,
track_alt_numbers,
track_context_alt_numbers,
direct_generated_rule_calls,
atn_preferred_rule_calls: &atn_preferred_rule_calls,
};
for rule in rules.iter().flatten() {
let index = rule.rule_index;
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
if rule.left_recursive {
writeln!(
out,
" self.parse_generated_rule_{index}_precedence(precedence, allow_fallback)"
)
.expect("writing to a string cannot fail");
} else {
writeln!(out, " let _ = precedence;").expect("writing to a string cannot fail");
writeln!(
out,
" self.parse_generated_rule_{index}(precedence, allow_fallback)"
)
.expect("writing to a string cannot fail");
}
writeln!(out, " }}").expect("writing to a string cannot fail");
render_generated_rule_method(&mut out, rule, step_render_context);
}
out
}
fn render_portable_local_declarations(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let Some(portable) = step_render_context.portable_locals else {
return;
};
let Some(declarations) = portable.declarations.get(rule_index) else {
return;
};
for declaration in declarations {
writeln!(out, " #[allow(unused_mut)]\n {declaration}")
.expect("writing to a string cannot fail");
}
}
/// Declares the embedded per-rule attrs local (`__attrs`) on rule entry.
fn render_embedded_attrs_local(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let Some(embedded) = step_render_context.embedded else {
return;
};
if !embedded
.rule_has_attrs
.get(rule_index)
.copied()
.unwrap_or_default()
{
return;
}
let attrs_struct = embedded::attrs_struct_name(rule_index);
writeln!(
out,
" #[allow(unused_mut)]\n let mut __attrs = {attrs_struct}::default();"
)
.expect("writing to a string cannot fail");
if let Some(arg0) = embedded
.rule_arg0
.get(rule_index)
.and_then(Option::as_deref)
{
writeln!(
out,
" if let Some(__arg) = self.__embedded_pending_arg.take() {{ __attrs.{arg0} = __arg as _; }}"
)
.expect("writing to a string cannot fail");
}
}
/// Runs the embedded `@init` body at rule entry, after context/attrs setup and
/// before matching the rule body.
fn render_embedded_init_entry(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let Some(embedded) = step_render_context.embedded else {
return;
};
if let Some(init) = embedded.init_entry.get(&rule_index) {
writeln!(out, " {init}").expect("writing to a string cannot fail");
}
}
/// Runs the embedded `@after` body (committed path only — ANTLR's caught-error
/// path skips `@after`) and seals the attrs snapshot before `finish_rule`.
fn render_embedded_after_and_seal(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
run_after: bool,
) {
let Some(embedded) = step_render_context.embedded else {
return;
};
if run_after {
if let Some(after) = embedded.after.get(&rule_index) {
writeln!(out, " {after}").expect("writing to a string cannot fail");
}
}
if embedded
.rule_has_attrs
.get(rule_index)
.copied()
.unwrap_or_default()
{
writeln!(
out,
" __ctx.set_generated_attrs(antlr4_runtime::GeneratedAttrs::new(__attrs.clone()));"
)
.expect("writing to a string cannot fail");
}
}
fn render_generated_rule_method(
out: &mut String,
rule: &GeneratedParserRule,
step_render_context: GeneratedStepRenderContext<'_>,
) {
if rule.left_recursive {
render_generated_left_recursive_rule_method(out, rule, step_render_context);
return;
}
let index = rule.rule_index;
let entry_state = rule.entry_state;
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}(&mut self, __precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " let _ = __precedence;").expect("writing to a string cannot fail");
writeln!(out, " let _ = allow_fallback;").expect("writing to a string cannot fail");
writeln!(
out,
" let __generated_diagnostic_marker = self.base.generated_diagnostics_checkpoint();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __ctx = self.base.enter_rule({entry_state}isize, {index});"
)
.expect("writing to a string cannot fail");
// Capture the rule start AFTER `enter_rule`, which advances the cursor past any
// leading hidden-channel tokens to the first visible token. Capturing before
// would make `$start`/`$text` in generated actions include a leading hidden
// prefix (e.g. whitespace), diverging from ANTLR and the rule context start.
writeln!(
out,
" let __rule_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_portable_local_declarations(out, index, step_render_context);
render_embedded_attrs_local(out, index, step_render_context);
render_embedded_init_entry(out, index, step_render_context);
writeln!(out, " let mut __consumed_eof = false;")
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __sync_error: Option<antlr4_runtime::AntlrError> = None;"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let __result = (|| -> Result<(), antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
render_generated_steps(out, &rule.steps, 3, step_render_context);
writeln!(out, " Ok(())").expect("writing to a string cannot fail");
writeln!(out, " }})();").expect("writing to a string cannot fail");
writeln!(out, " match __result {{").expect("writing to a string cannot fail");
writeln!(out, " Ok(()) => {{").expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, true);
writeln!(
out,
" let __tree = self.base.finish_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " Err(__error) => {{").expect("writing to a string cannot fail");
// A rule's own `sync_decision` failure (`__sync_error`) is fatal ONLY at the
// top-level public entry (`allow_fallback`). When this rule is a nested child
// (`!allow_fallback`), ANTLR recovers the mismatch INSIDE the child and returns
// a partial subtree to the parent — it never propagates the sync failure up. So
// for a nested child, recover locally like any other body error (a `Fatal`
// escaping here would make the parent recover on ITS context, dropping the
// child subtree). Only the true top-level keeps the `Fatal` abort (preserving
// antlr#6 `InvalidEmptyInput`-style start-rule errors).
writeln!(
out,
" if let Some(__error) = __sync_error {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " if allow_fallback {{")
.expect("writing to a string cannot fail");
writeln!(out, " self.base.exit_rule();")
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.restore_generated_diagnostics(__generated_diagnostic_marker);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.record_generated_syntax_error();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" return Err(GeneratedRuleError::Fatal(__error));"
)
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " return Ok(__tree);")
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
}
fn render_generated_left_recursive_rule_method(
out: &mut String,
rule: &GeneratedParserRule,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let index = rule.rule_index;
let entry_state = rule.entry_state;
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}(&mut self, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.parse_generated_rule_{index}_precedence(0, allow_fallback)"
)
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}_precedence(&mut self, __precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " let _ = allow_fallback;").expect("writing to a string cannot fail");
writeln!(
out,
" let __generated_diagnostic_marker = self.base.generated_diagnostics_checkpoint();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __ctx = self.base.enter_recursion_rule({entry_state}isize, {index}, __precedence);"
)
.expect("writing to a string cannot fail");
// Capture the rule start AFTER `enter_recursion_rule`, which (via `enter_rule`)
// advances the cursor past any leading hidden-channel tokens to the first
// visible token. Capturing before would make `$start`/`$text` in generated
// actions include a leading hidden prefix, diverging from ANTLR.
writeln!(
out,
" let __rule_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_portable_local_declarations(out, index, step_render_context);
render_embedded_attrs_local(out, index, step_render_context);
render_embedded_init_entry(out, index, step_render_context);
writeln!(out, " let mut __consumed_eof = false;")
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __sync_error: Option<antlr4_runtime::AntlrError> = None;"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let __result = (|| -> Result<(), antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
render_generated_steps(out, &rule.steps, 3, step_render_context);
writeln!(out, " Ok(())").expect("writing to a string cannot fail");
writeln!(out, " }})();").expect("writing to a string cannot fail");
writeln!(out, " match __result {{").expect("writing to a string cannot fail");
writeln!(out, " Ok(()) => {{").expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, true);
writeln!(
out,
" let __tree = self.base.finish_recursion_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " Err(__error) => {{").expect("writing to a string cannot fail");
// Same as the non-left-recursive case: a nested child (`!allow_fallback`)
// recovers its own sync failure internally and returns a partial subtree; only
// the top-level entry propagates `Fatal`. Use `finish_recursion_rule` (which
// unrolls the recursion context) in the recover branch — do NOT also call
// `unroll_recursion_context` (that would double-unroll).
writeln!(
out,
" if let Some(__error) = __sync_error {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " if allow_fallback {{")
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.unroll_recursion_context();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.restore_generated_diagnostics(__generated_diagnostic_marker);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.record_generated_syntax_error();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" return Err(GeneratedRuleError::Fatal(__error));"
)
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_recursion_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " return Ok(__tree);")
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_recursion_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
}
fn render_generated_steps(
out: &mut String,
steps: &[GeneratedParserStep],
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
for step in steps {
render_generated_step(out, step, indent, render_context);
}
}
fn render_generated_step(
out: &mut String,
step: &GeneratedParserStep,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let pad = " ".repeat(indent);
match step {
GeneratedParserStep::MatchToken {
token_type,
follow_state,
} => {
writeln!(
out,
"{pad}let __match = self.base.match_token_recovering({token_type}, {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::MatchSet {
token_set,
intervals,
follow_state,
} => {
if let Some(token_set) = token_set {
writeln!(
out,
"{pad}let __match = self.base.match_token_set_recovering(atn().token_set({token_set}).expect(\"generated parser token-set index\"), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
} else {
let intervals = render_i32_ranges(intervals);
writeln!(
out,
"{pad}let __match = self.base.match_set_recovering(&{intervals}, {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
}
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::MatchNotSet {
token_set,
intervals,
follow_state,
} => {
if let Some(token_set) = token_set {
writeln!(
out,
"{pad}let __match = self.base.match_not_token_set_recovering(atn().token_set({token_set}).expect(\"generated parser token-set index\"), 1, atn().max_token_type(), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
} else {
let intervals = render_i32_ranges(intervals);
writeln!(
out,
"{pad}let __match = self.base.match_not_set_recovering(&{intervals}, 1, atn().max_token_type(), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
}
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::MatchWildcard { follow_state } => {
// A wildcard matches any single token. Model it as a not-set with an
// empty exclusion set (every token in 1..=max), reusing the recovering
// match so a wildcard at EOF performs ANTLR's single-token insertion
// (`<missing ...>` error node) and lets the rule continue, instead of
// aborting the remaining steps.
writeln!(
out,
"{pad}let __match = self.base.match_not_set_recovering(&[], 1, atn().max_token_type(), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::Precedence(precedence) => {
writeln!(out, "{pad}if !self.base.precpred({precedence}) {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"precpred(_ctx, {precedence})\"));"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
GeneratedParserStep::Predicate {
rule_index,
pred_index,
} => {
if let Some((condition, message)) = render_context
.portable_locals
.and_then(|portable| portable.predicates.get(&(*rule_index, *pred_index)))
{
writeln!(out, "{pad}if !({condition}) {{")
.expect("writing to a string cannot fail");
match message {
Some(message) => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_option_error({rule_index}, \"{}\".to_owned()));",
rust_string(message)
)
.expect("writing to a string cannot fail"),
None => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"semantic predicate\"));"
)
.expect("writing to a string cannot fail"),
}
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
return;
}
if let Some(embedded) = render_context.embedded {
let (condition, message) =
embedded_predicate_condition_and_message(&embedded, *rule_index, *pred_index);
writeln!(out, "{pad}if !({condition}) {{")
.expect("writing to a string cannot fail");
match message {
Some(message) => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_option_error({rule_index}, \"{}\".to_owned()));",
rust_string(&message)
)
.expect("writing to a string cannot fail"),
None => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"semantic predicate\"));"
)
.expect("writing to a string cannot fail"),
}
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
return;
}
writeln!(
out,
"{pad}if !self.base.parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), {rule_index}, {pred_index}, &__ctx, __precedence) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} if let Some(__message) = self.base.parser_semantic_ir_predicate_failure_message({rule_index}, {pred_index}, parser_semantics()) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} return Err(self.base.failed_predicate_option_error({rule_index}, __message));"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"semantic predicate\"));"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
GeneratedParserStep::CallRule {
source_state,
rule_index,
precedence,
} => {
writeln!(
out,
"{pad}let __invoking_marker = self.base.push_invoking_state({source_state}isize);"
)
.expect("writing to a string cannot fail");
if let Some(embedded) = render_context.embedded {
if let Some(expression) = embedded.call_args.get(source_state) {
writeln!(
out,
"{pad}self.__embedded_pending_arg = Some(i64::from({expression}));"
)
.expect("writing to a string cannot fail");
}
}
let precedence = match precedence {
GeneratedRuleCallPrecedence::Literal(value) => value.to_string(),
GeneratedRuleCallPrecedence::InheritLocal => "__precedence".to_owned(),
};
let from_generated_call =
format!("self.parse_rule_precedence_from_generated({rule_index}, {precedence})");
let generated_child_call = if render_context
.direct_generated_rule_calls
.get(*rule_index)
.copied()
.unwrap_or_default()
{
format!(
"self.parse_generated_rule_{rule_index}_dispatch({precedence}, false).map_err(GeneratedRuleError::into_error)"
)
} else {
from_generated_call.clone()
};
let child_call = if render_context
.atn_preferred_rule_calls
.get(*rule_index)
.copied()
.unwrap_or_default()
{
// ATN-preferred child: route through `parse_rule_precedence_from_generated`.
// The rule's `parse_generated_rule` dispatch arm is guarded by
// `generated_only()`, so in normal mode the generated probe returns
// `None` and the wrapper parses the child on the INTERPRETED path
// (preserving the ATN-preferred optimization).
from_generated_call
} else {
generated_child_call
};
writeln!(out, "{pad}let __child = {child_call};")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}self.base.discard_invoking_state(__invoking_marker);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}let __child = __child?;").expect("writing to a string cannot fail");
writeln!(out, "{pad}self.base.add_parse_child(&mut __ctx, __child);")
.expect("writing to a string cannot fail");
}
GeneratedParserStep::Action {
source_state,
rule_index,
} => {
writeln!(
out,
"{pad}let action = self.base.parser_action_at_current({source_state}, {rule_index}, __rule_start, __consumed_eof);"
)
.expect("writing to a string cannot fail");
if let Some(statement) = render_context.inline_action_statements.get(source_state) {
if !statement.is_empty() {
writeln!(out, "{pad}{statement}").expect("writing to a string cannot fail");
}
}
if render_context.embedded.is_some() {
// Embedded actions executed inline just above, at their
// ANTLR-correct point in the rule body.
writeln!(out, "{pad}let _ = &action;").expect("writing to a string cannot fail");
return;
}
writeln!(out, "{pad}let _ = action;").expect("writing to a string cannot fail");
}
GeneratedParserStep::Decision {
state,
decision,
track_alt_number,
allow_semantic_context,
force_context,
fast_path,
alts,
} => {
render_generated_decision(
out,
DecisionRender {
state: *state,
decision: *decision,
track_alt_number: *track_alt_number,
allow_semantic_context: *allow_semantic_context,
force_context: *force_context,
fast_path: fast_path.as_ref(),
alts,
},
indent,
render_context,
);
}
GeneratedParserStep::StarLoop {
state,
decision,
enter_alt,
exit_alt,
track_alt_number,
allow_semantic_context,
force_context,
plus_loop,
fast_path,
body,
} => {
render_generated_star_loop(
out,
StarLoopRender {
state: *state,
decision: *decision,
alts: (*enter_alt, *exit_alt),
track_alt_number: *track_alt_number,
allow_semantic_context: *allow_semantic_context,
force_context: *force_context,
plus_loop: *plus_loop,
fast_path: fast_path.as_ref(),
body,
},
indent,
render_context,
);
}
GeneratedParserStep::LeftRecursiveLoop {
state,
decision,
enter_alt,
exit_alt,
rule_index,
entry_state,
body,
..
} => {
render_generated_left_recursive_loop(
out,
LeftRecursiveLoopRender {
state: *state,
decision: *decision,
alts: (*enter_alt, *exit_alt),
rule: (*rule_index, *entry_state),
body,
},
indent,
render_context,
);
}
}
}
fn render_generated_decision(
out: &mut String,
decision_info: DecisionRender<'_>,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let DecisionRender {
state,
decision,
track_alt_number,
allow_semantic_context,
force_context,
fast_path,
alts,
} = decision_info;
let pad = " ".repeat(indent);
// A tool-LL(1) decision dispatches on the tool's complete LOOK table
// (exit alternatives included), like Java's switch compilation.
let tool_fast_path = render_context
.embedded
.and_then(|embedded| embedded.tool_ll1_fast_path(decision));
let fast_path = tool_fast_path.as_ref().or(fast_path);
if let Some(fast_path) = fast_path.filter(|_| {
!allow_semantic_context
&& !force_context
&& !render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision))
}) {
writeln!(
out,
"{pad}let mut __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}let __prediction = match self.base.la(1) {{")
.expect("writing to a string cannot fail");
render_generated_fast_prediction_arms(out, &pad, fast_path);
writeln!(out, "{pad} _ => {{").expect("writing to a string cannot fail");
// A non-loop block/optional decision is never a loop-back: ANTLR syncs it
// like BLOCK_START (single-token deletion), so pass `false`.
render_generated_sync_decision(out, &format!("{pad} "), state, "false");
writeln!(
out,
"{pad} __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_generated_ll1_then_adaptive_prediction(
out,
&format!("{pad} "),
state,
decision,
false,
);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
} else {
if !allow_semantic_context {
render_generated_sync_decision(out, &pad, state, "false");
}
writeln!(
out,
"{pad}let __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
let force_adaptive = render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision));
if allow_semantic_context || force_context {
render_generated_adaptive_prediction(out, &pad, decision);
} else if force_adaptive {
render_generated_two_stage_adaptive_assignment(out, &pad, decision);
} else {
render_generated_ll1_then_adaptive_prediction(out, &pad, state, decision, true);
}
}
if allow_semantic_context {
render_generated_semantic_prediction_filter(
out,
&pad,
alts,
render_context.embedded,
render_context.portable_locals,
);
render_generated_decision_diagnostic_report(
out,
&pad,
state,
alts,
render_context.embedded,
render_context.portable_locals,
);
} else {
writeln!(
out,
"{pad}self.base.record_generated_prediction_diagnostic(atn(), {state}, &__prediction);"
)
.expect("writing to a string cannot fail");
}
writeln!(out, "{pad}match __prediction.alt {{").expect("writing to a string cannot fail");
for (index, steps) in alts.iter().enumerate() {
let alt = index + 1;
writeln!(out, "{pad} {alt} => {{").expect("writing to a string cannot fail");
render_generated_alt_number_assignments(
out,
&format!("{pad} "),
alt,
render_context.track_alt_numbers && track_alt_number,
render_context.track_context_alt_numbers && track_alt_number,
);
render_generated_steps(out, steps, indent + 2, render_context);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(
out,
"{pad} _ => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_fast_prediction_arms(
out: &mut String,
pad: &str,
fast_path: &GeneratedDecisionFastPath,
) {
for arm in &fast_path.arms {
let patterns = render_i32_match_patterns(&arm.intervals);
let alt = arm.alt;
writeln!(
out,
"{pad} {patterns} => antlr4_runtime::ParserAtnPrediction {{ alt: {alt}, requires_full_context: false, has_semantic_context: false, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
}
}
/// Two-stage adaptive prediction without the LL(1) shortcut — Java's plain
/// `adaptivePredict`: the SLL probe resolves or flags a full-context
/// conflict, and the retry with real outer context only runs when the
/// parser's prediction mode allows it (never in SLL mode).
fn render_generated_two_stage_adaptive_assignment(out: &mut String, pad: &str, decision: usize) {
writeln!(out, "{pad}let __prediction = {{").expect("writing to a string cannot fail");
render_generated_sll_then_context_prediction_with_indent(out, pad, decision, 1);
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn render_generated_ll1_then_adaptive_prediction(
out: &mut String,
pad: &str,
state: usize,
decision: usize,
assign: bool,
) {
let prefix = if assign { "let __prediction = " } else { "" };
let suffix = if assign { ";" } else { "" };
writeln!(
out,
"{pad}{prefix}if let Some(__prediction) = self.base.ll1_decision_prediction(atn(), {state}) {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad}}} else {{").expect("writing to a string cannot fail");
render_generated_sll_then_context_prediction_with_indent(out, pad, decision, 1);
writeln!(out, "{pad}}}{suffix}").expect("writing to a string cannot fail");
}
fn render_generated_decision_diagnostic_report(
out: &mut String,
pad: &str,
state: usize,
alts: &[Vec<GeneratedParserStep>],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) {
let alt_conditions = alts
.iter()
.map(|steps| {
semantic_alt_candidate_condition_with_la(steps, "__diagnostic_la", embedded, portable)
})
.collect::<Vec<_>>();
if alt_conditions
.iter()
.any(|condition| condition == "true" || condition == "false")
{
return;
}
writeln!(out, "{pad}if self.base.report_diagnostic_errors() {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __diagnostic_la = self.base.la(1);")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let mut __diagnostic_alts = Vec::new();")
.expect("writing to a string cannot fail");
for (index, condition) in alt_conditions.iter().enumerate() {
let alt = index + 1;
writeln!(out, "{pad} if {condition} {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __diagnostic_alts.push({alt});")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(
out,
"{pad} self.base.record_generated_ambiguity_diagnostic(atn(), {state}, __decision_start, __decision_start, &__diagnostic_alts);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_semantic_prediction_filter(
out: &mut String,
pad: &str,
alts: &[Vec<GeneratedParserStep>],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) {
let alt_has_predicates = alts
.iter()
.map(|steps| !leading_predicates(steps, portable).is_empty())
.collect::<Vec<_>>();
if !alt_has_predicates
.iter()
.any(|has_predicate| *has_predicate)
{
return;
}
let alt_conditions = alts
.iter()
.map(|steps| semantic_alt_candidate_condition(steps, embedded, portable))
.collect::<Vec<_>>();
writeln!(
out,
"{pad}let __prediction = if __prediction.has_semantic_context {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __semantic_la = self.base.la(1);")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __semantic_alt = match __prediction.alt {{"
)
.expect("writing to a string cannot fail");
for (index, condition) in alt_conditions.iter().enumerate() {
if !alt_has_predicates[index] {
continue;
}
let alt = index + 1;
writeln!(out, "{pad} {alt} if {condition} => Some({alt}),")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} {alt} => {{").expect("writing to a string cannot fail");
render_semantic_alt_search(out, pad, &alt_conditions, alts, portable);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(out, "{pad} _ => Some(__prediction.alt),")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }};").expect("writing to a string cannot fail");
writeln!(out, "{pad} match __semantic_alt {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Some(__alt) => antlr4_runtime::ParserAtnPrediction {{ alt: __alt, ..__prediction }},"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} None => {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __error = self.base.no_viable_alternative_error(__decision_start);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} return Err(__error);")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}} else {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn render_semantic_alt_search(
out: &mut String,
pad: &str,
alt_conditions: &[String],
alts: &[Vec<GeneratedParserStep>],
portable: Option<PortableLocalStepRender<'_>>,
) {
// The predicted alt's predicate failed; pick another alt whose candidate
// condition holds. This runs in TWO passes so an alt whose viability is not
// locally checkable (no predicate and no computable lookahead — its first
// consuming step is a rule call/decision/loop, giving condition `"true"`)
// neither shadows a concretely-guarded alt nor becomes unreachable:
//
// Pass 1 — alts with a resolved guard (predicate and/or lookahead), in
// order. A later token-led alt is reachable even if an earlier
// unresolved rule-call alt exists (`{p()}? 'a' | x | 'a'` on input `a`
// picks the `'a'` alt, not rule-call `x`).
// Pass 2 — the remaining unresolved alts, in order, as a last resort. So
// an unresolved alt is still tried when no resolved alt matched
// (`{p()}? 'a' | x` on input in FIRST(x) selects `x` instead of
// reporting NoViableAlt).
//
// Scoped to the fallback search only; the shared
// `semantic_alt_candidate_condition` is unchanged, so left-recursion
// loop-entry and diagnostic paths keep their behavior.
let unresolved = alts
.iter()
.map(|steps| semantic_alt_guard_is_unresolved(steps, portable))
.collect::<Vec<_>>();
for (index, condition) in alt_conditions.iter().enumerate() {
if unresolved.get(index).copied().unwrap_or(false) {
continue;
}
let alt = index + 1;
writeln!(out, "{pad} if {condition} {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} Some({alt})").expect("writing to a string cannot fail");
writeln!(out, "{pad} }} else").expect("writing to a string cannot fail");
}
// Last-resort pass: unresolved alts keep their real condition (typically
// `true`), so they are tried only after every resolved alt missed.
for (index, condition) in alt_conditions.iter().enumerate() {
if !unresolved.get(index).copied().unwrap_or(false) {
continue;
}
let alt = index + 1;
writeln!(out, "{pad} if {condition} {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} Some({alt})").expect("writing to a string cannot fail");
writeln!(out, "{pad} }} else").expect("writing to a string cannot fail");
}
writeln!(out, "{pad} {{ None }}").expect("writing to a string cannot fail");
}
/// Looks up the verbatim predicate expression (and optional `<fail=…>`
/// message) for a coordinate in embedded mode. A coordinate with no embedded
/// body evaluates true, matching ANTLR's treatment of a missing predicate.
fn embedded_predicate_condition_and_message(
embedded: &EmbeddedStepRender<'_>,
rule_index: usize,
pred_index: usize,
) -> (String, Option<String>) {
embedded
.predicates
.get(&(rule_index, pred_index))
.map_or_else(
|| ("true".to_owned(), None),
|(expression, message)| (expression.clone(), message.clone()),
)
}
fn semantic_alt_candidate_condition(
steps: &[GeneratedParserStep],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) -> String {
semantic_alt_candidate_condition_with_la(steps, "__semantic_la", embedded, portable)
}
fn semantic_alt_candidate_condition_with_la(
steps: &[GeneratedParserStep],
la_symbol: &str,
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) -> String {
// Order matters: the lookahead guard comes FIRST so `&&` short-circuits on it
// before any predicate hook runs. Otherwise, searching alternatives in a
// semantic decision would evaluate an alternative's leading hook/unknown
// predicate even when its first token cannot match the current lookahead —
// recording a spurious fail-loud `Unsupported` hit under
// `--sem-unknown=hook`/`error` and rejecting a later syntactically viable
// alternative. This also matches ANTLR, which only evaluates a predicate for
// a lookahead-viable alternative.
let mut conditions = Vec::new();
if let Some(lookahead) = leading_lookahead_condition(steps, la_symbol) {
conditions.push(lookahead);
}
conditions.extend(leading_predicates(steps, portable).into_iter().map(
|(rule_index, pred_index)| {
if let Some((condition, _)) =
portable.and_then(|portable| portable.predicates.get(&(rule_index, pred_index)))
{
return format!("({condition})");
}
embedded.map_or_else(
|| {
format!(
"self.base.parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), {rule_index}, {pred_index}, &__ctx, __precedence)"
)
},
|embedded| {
let (condition, _) = embedded_predicate_condition_and_message(
&embedded, rule_index, pred_index,
);
format!("({condition})")
},
)
},
));
if conditions.is_empty() {
"true".to_owned()
} else {
conditions.join(" && ")
}
}
/// Whether an alternative has no locally-checkable viability guard: no leading
/// predicate and no computable lookahead (its first consuming step is a
/// `CallRule` / nested decision / loop whose FIRST set is not computed here).
/// Such an alt's [`semantic_alt_candidate_condition`] is `"true"`, so in the
/// ordered semantic-alt fallback search it would shadow a later alt with a
/// concrete matching lookahead. The search treats these as last-resort
/// candidates instead. A genuine epsilon alt (no consuming step at all) is NOT
/// unguarded in this sense — it legitimately matches anything.
fn semantic_alt_guard_is_unresolved(
steps: &[GeneratedParserStep],
portable: Option<PortableLocalStepRender<'_>>,
) -> bool {
if !leading_predicates(steps, portable).is_empty() {
return false;
}
if leading_lookahead_condition(steps, "__semantic_la").is_some() {
return false;
}
// No predicate and no computable lookahead: unresolved only if a consuming
// step exists (rule call / decision / loop). Pure epsilon stays resolved.
steps.iter().any(|step| {
matches!(
step,
GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::Decision { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. }
)
})
}
fn leading_predicates(
steps: &[GeneratedParserStep],
portable: Option<PortableLocalStepRender<'_>>,
) -> Vec<(usize, usize)> {
let mut predicates = Vec::new();
for step in steps {
match step {
GeneratedParserStep::Predicate {
rule_index,
pred_index,
} => predicates.push((*rule_index, *pred_index)),
// Portable assignments run only after the alternative is selected,
// so predicates after one are not prediction-visible.
GeneratedParserStep::Action { source_state, .. }
if portable
.is_some_and(|portable| portable.inline_actions.contains_key(source_state)) =>
{
break;
}
GeneratedParserStep::Action { .. } | GeneratedParserStep::Precedence(_) => {}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::Decision { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => break,
}
}
predicates
}
fn leading_lookahead_condition(steps: &[GeneratedParserStep], la_symbol: &str) -> Option<String> {
for step in steps {
match step {
GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::Precedence(_) => {}
GeneratedParserStep::MatchToken { token_type, .. } => {
return Some(format!("{la_symbol} == {token_type}"));
}
GeneratedParserStep::MatchSet {
token_set,
intervals,
..
} => {
return Some(token_set_condition(la_symbol, *token_set, intervals));
}
GeneratedParserStep::MatchNotSet {
token_set,
intervals,
..
} => {
let excluded = token_set_condition(la_symbol, *token_set, intervals);
return Some(format!(
"(1..=atn().max_token_type()).contains(&{la_symbol}) && !({excluded})"
));
}
GeneratedParserStep::MatchWildcard { .. } => {
return Some(format!("{la_symbol} != antlr4_runtime::TOKEN_EOF"));
}
GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::Decision { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => return None,
}
}
None
}
fn token_set_condition(symbol: &str, token_set: Option<usize>, intervals: &[(i32, i32)]) -> String {
token_set.map_or_else(
|| intervals_condition(symbol, intervals),
|token_set| {
format!(
"atn().token_set({token_set}).expect(\"generated parser token-set index\").contains({symbol})"
)
},
)
}
fn intervals_condition(symbol: &str, intervals: &[(i32, i32)]) -> String {
if intervals.is_empty() {
return "false".to_owned();
}
intervals
.iter()
.map(|(start, stop)| {
if start == stop {
format!("{symbol} == {start}")
} else {
format!("({start}..={stop}).contains(&{symbol})")
}
})
.collect::<Vec<_>>()
.join(" || ")
}
fn render_generated_alt_number_assignments(
out: &mut String,
pad: &str,
alt: usize,
track_alt_number: bool,
track_context_alt_number: bool,
) {
if track_alt_number {
writeln!(out, "{pad}if __ctx.alt_number() == 0 {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} __ctx.set_alt_number({alt});")
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
if track_context_alt_number {
writeln!(out, "{pad}if __ctx.context_alt_number() == 0 {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} __ctx.set_context_alt_number({alt});")
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
}
fn render_generated_sync_decision(out: &mut String, pad: &str, state: usize, loop_back_expr: &str) {
writeln!(
out,
"{pad}match self.base.sync_decision(atn(), {state}, !__ctx.has_matched_child(), {loop_back_expr}) {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} Ok(__sync_children) => {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} for __child in __sync_children {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} Err(__error) => {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __sync_error = Some(__error.clone());")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} return Err(__error);").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_adaptive_prediction(out: &mut String, pad: &str, decision: usize) {
writeln!(out, "{pad}let __prediction = {{").expect("writing to a string cannot fail");
render_generated_adaptive_prediction_with_indent(out, pad, decision, 1);
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn render_generated_adaptive_prediction_with_indent(
out: &mut String,
pad: &str,
decision: usize,
extra_indent: usize,
) {
let nested = format!("{pad}{}", " ".repeat(extra_indent));
writeln!(
out,
"{nested}let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested}let __prediction_context = __simulator.intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested}__simulator.set_exact_ambig_detection(self.base.prediction_mode() == antlr4_runtime::PredictionMode::LlExactAmbigDetection);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested}__simulator.adaptive_predict_stream_info_with_context({decision}, 0, self.base.input(), __prediction_context)"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} .map_err(|__error| match __error {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ index, .. }} => self.base.no_viable_alternative_error_at(__decision_start, index),"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} _ => self.base.no_viable_alternative_error(__decision_start),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} }})?").expect("writing to a string cannot fail");
}
fn render_generated_sll_then_context_prediction_with_indent(
out: &mut String,
pad: &str,
decision: usize,
extra_indent: usize,
) {
let nested = format!("{pad}{}", " ".repeat(extra_indent));
writeln!(out, "{nested}let __prediction = {{").expect("writing to a string cannot fail");
writeln!(
out,
"{nested} let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
// Stage 1 uses the SLL probe: on a full-context-requiring conflict it returns
// requires_full_context WITHOUT running the LL loop (the result is discarded
// here anyway — only the boolean gates the stage-2 re-run with real context).
writeln!(
out,
"{nested} __simulator.adaptive_predict_stream_info_sll_probe({decision}, 0, self.base.input())"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} .map_err(|__error| match __error {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ index, .. }} => self.base.no_viable_alternative_error_at(__decision_start, index),"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} _ => self.base.no_viable_alternative_error(__decision_start),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} }})?").expect("writing to a string cannot fail");
writeln!(out, "{nested}}};").expect("writing to a string cannot fail");
writeln!(
out,
"{nested}if __prediction.requires_full_context && self.base.prediction_mode() != antlr4_runtime::PredictionMode::Sll {{"
)
.expect("writing to a string cannot fail");
render_generated_adaptive_prediction_with_indent(out, pad, decision, extra_indent + 1);
writeln!(out, "{nested}}} else {{").expect("writing to a string cannot fail");
writeln!(out, "{nested} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{nested}}}").expect("writing to a string cannot fail");
}
fn render_generated_star_loop(
out: &mut String,
loop_info: StarLoopRender<'_>,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let StarLoopRender {
state,
decision,
alts,
track_alt_number,
allow_semantic_context,
force_context,
plus_loop,
fast_path,
body,
} = loop_info;
let (enter_alt, exit_alt) = alts;
let pad = " ".repeat(indent);
// A tool-LL(1) loop decision dispatches on the tool's complete LOOK
// table (exit alternative included), like Java's switch-driven loops.
let tool_fast_path = render_context
.embedded
.and_then(|embedded| embedded.tool_ll1_fast_path(decision));
let fast_path = tool_fast_path.as_ref().or(fast_path);
// Per-loop "iteration started" flag, threaded into `sync_decision` so it
// recovers like ANTLR: a `*` loop's first sync is at the loop ENTRY
// (single-token deletion), every later sync is a loop-BACK (multi-token
// `consumeUntil`). A `+` loop's mandatory first element is iteration 1, so it
// is already on the loop-back side at its first sync (init `true`).
let loop_iter = format!("__loop_iter_{state}");
writeln!(out, "{pad}let mut {loop_iter} = {plus_loop};")
.expect("writing to a string cannot fail");
writeln!(out, "{pad}loop {{").expect("writing to a string cannot fail");
let inner_pad = format!("{pad} ");
if let Some(fast_path) = fast_path.filter(|_| {
!allow_semantic_context
&& !force_context
&& !render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision))
}) {
writeln!(
out,
"{pad} let mut __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __prediction = match self.base.la(1) {{")
.expect("writing to a string cannot fail");
render_generated_fast_prediction_arms(out, &inner_pad, fast_path);
writeln!(out, "{pad} _ => {{").expect("writing to a string cannot fail");
render_generated_sync_decision(out, &format!("{pad} "), state, &loop_iter);
writeln!(
out,
"{pad} __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_generated_ll1_then_adaptive_prediction(
out,
&format!("{pad} "),
state,
decision,
false,
);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} }};").expect("writing to a string cannot fail");
} else {
render_generated_sync_decision(out, &inner_pad, state, &loop_iter);
writeln!(
out,
"{pad} let __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
let force_adaptive = render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision));
if allow_semantic_context || force_context {
render_generated_adaptive_prediction(out, &inner_pad, decision);
} else if force_adaptive {
render_generated_two_stage_adaptive_assignment(out, &inner_pad, decision);
} else {
render_generated_ll1_then_adaptive_prediction(out, &inner_pad, state, decision, true);
}
}
render_generated_loop_semantic_prediction_filter(
out,
&format!("{pad} "),
enter_alt,
exit_alt,
body,
render_context.embedded,
render_context.portable_locals,
);
writeln!(
out,
"{pad} self.base.record_generated_prediction_diagnostic(atn(), {state}, &__prediction);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} match __prediction.alt {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} {enter_alt} => {{").expect("writing to a string cannot fail");
// Once an iteration is taken, every subsequent sync is a loop-back.
writeln!(out, "{pad} {loop_iter} = true;").expect("writing to a string cannot fail");
render_generated_alt_number_assignments(
out,
&format!("{pad} "),
enter_alt,
render_context.track_alt_numbers && track_alt_number,
render_context.track_context_alt_numbers && track_alt_number,
);
render_generated_steps(out, body, indent + 3, render_context);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} {exit_alt} => {{").expect("writing to a string cannot fail");
render_generated_alt_number_assignments(
out,
&format!("{pad} "),
exit_alt,
render_context.track_alt_numbers && track_alt_number,
render_context.track_context_alt_numbers && track_alt_number,
);
writeln!(out, "{pad} break;").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} _ => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_left_recursive_loop(
out: &mut String,
loop_info: LeftRecursiveLoopRender<'_>,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let LeftRecursiveLoopRender {
state,
decision,
alts,
rule,
body,
} = loop_info;
let (rule_index, entry_state) = rule;
let (enter_alt, exit_alt) = alts;
let pad = " ".repeat(indent);
writeln!(out, "{pad}loop {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
if render_context.embedded.is_some() {
writeln!(
out,
"{pad} let __prediction_precedence = if __precedence <= 0 {{ 0 }} else {{ __precedence as usize }};"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __prediction = match {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __prediction_context = __simulator.intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} __simulator.set_exact_ambig_detection(self.base.prediction_mode() == antlr4_runtime::PredictionMode::LlExactAmbigDetection);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} __simulator.adaptive_predict_stream_info_with_context({decision}, __prediction_precedence, self.base.input(), __prediction_context)"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }} {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} Ok(__prediction) => __prediction,")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ .. }}) if self.base.left_recursive_loop_enter_matches(atn(), {state}, __precedence) => {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} antlr4_runtime::ParserAtnPrediction {{ alt: {enter_alt}, requires_full_context: true, has_semantic_context: true, diagnostic: None }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ .. }}) => {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, requires_full_context: true, has_semantic_context: false, diagnostic: None }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(_) => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
} else {
writeln!(
out,
"{pad} let __prediction = match self.base.left_recursive_loop_enter_prediction(atn(), {state}, __precedence) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Some(true) => antlr4_runtime::ParserAtnPrediction {{ alt: {enter_alt}, requires_full_context: false, has_semantic_context: true, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Some(false) => antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, requires_full_context: false, has_semantic_context: false, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} None => {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __prediction_precedence = if __precedence <= 0 {{ 0 }} else {{ __precedence as usize }};"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __prediction_context = __simulator.intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} __simulator.set_exact_ambig_detection(self.base.prediction_mode() == antlr4_runtime::PredictionMode::LlExactAmbigDetection);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} match __simulator.adaptive_predict_stream_info_with_context({decision}, __prediction_precedence, self.base.input(), __prediction_context) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Ok(__prediction) => __prediction,"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ .. }}) => antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, requires_full_context: true, has_semantic_context: false, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(_) => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(out, "{pad} }};").expect("writing to a string cannot fail");
render_generated_loop_semantic_prediction_filter(
out,
&format!("{pad} "),
enter_alt,
exit_alt,
body,
render_context.embedded,
render_context.portable_locals,
);
writeln!(
out,
"{pad} self.base.record_generated_prediction_diagnostic(atn(), {state}, &__prediction);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} match __prediction.alt {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} {enter_alt} => {{").expect("writing to a string cannot fail");
if render_context.embedded.is_some_and(|embedded| {
embedded
.rule_has_attrs
.get(rule_index)
.copied()
.unwrap_or(false)
}) {
// Java's `_prevctx`: the outgoing iteration context becomes the new
// context's left child, so it must carry the attrs accumulated so
// far — operator-alt actions and typed views read them through the
// child context, not the live `__attrs` local.
writeln!(
out,
"{pad} __ctx.set_generated_attrs(antlr4_runtime::GeneratedAttrs::new(__attrs.clone()));"
)
.expect("writing to a string cannot fail");
}
writeln!(
out,
"{pad} self.base.push_new_recursion_context_with_previous({entry_state}isize, {rule_index}, &mut __ctx);"
)
.expect("writing to a string cannot fail");
render_generated_steps(out, body, indent + 3, render_context);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} {exit_alt} => break,").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} _ => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
#[allow(clippy::too_many_arguments)]
fn render_generated_loop_semantic_prediction_filter(
out: &mut String,
pad: &str,
enter_alt: usize,
exit_alt: usize,
body: &[GeneratedParserStep],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) {
let Some(condition) = loop_entry_condition(body, embedded, portable) else {
return;
};
writeln!(
out,
"{pad}let __prediction = if __prediction.alt == {enter_alt} {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __semantic_la = self.base.la(1);")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} if {condition} {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad} }} else {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, ..__prediction }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}} else {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn loop_entry_condition(
body: &[GeneratedParserStep],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) -> Option<String> {
let step = body.first()?;
match step {
GeneratedParserStep::Predicate { .. } | GeneratedParserStep::Precedence(_) => {
Some(semantic_alt_candidate_condition(body, embedded, portable))
}
GeneratedParserStep::Decision { alts, .. } => {
if !alts.iter().any(|alt| steps_contain_predicate(alt)) {
return None;
}
Some(
alts.iter()
.map(|alt| {
format!(
"({})",
semantic_alt_candidate_condition(alt, embedded, portable)
)
})
.collect::<Vec<_>>()
.join(" || "),
)
}
GeneratedParserStep::Action { .. }
| GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => None,
}
}
#[allow(clippy::fn_params_excessive_bools)]
fn render_parser_parse_rule_fallback(
track_alt_numbers: bool,
track_context_alt_numbers: bool,
rule_args: &[(usize, usize, RuleArgTemplate)],
has_action_dispatch: bool,
has_predicate_dispatch: bool,
unknown_policy_literal: Option<&str>,
) -> String {
let mut out = String::new();
if has_predicate_dispatch || unknown_policy_literal.is_some() {
writeln!(
out,
"let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ track_alt_numbers: {track_alt_numbers}, track_context_alt_numbers: {track_context_alt_numbers}, predicates: &[], semantics: Some(parser_semantics()), rule_args: &{}, member_actions: &[], return_actions: &[], unknown_predicate_policy: {} , ..antlr4_runtime::ParserRuntimeOptions::default() }})?;",
render_parser_rule_arg_array(rule_args),
unknown_policy_literal
.unwrap_or("antlr4_runtime::UnknownSemanticPolicy::AssumeTrue")
)
.expect("writing to a string cannot fail");
} else if track_alt_numbers || track_context_alt_numbers {
writeln!(
out,
"let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ track_alt_numbers: {track_alt_numbers}, track_context_alt_numbers: {track_context_alt_numbers}, ..antlr4_runtime::ParserRuntimeOptions::default() }})?;"
)
.expect("writing to a string cannot fail");
} else if has_action_dispatch {
writeln!(
out,
"let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions::default())?;"
)
.expect("writing to a string cannot fail");
} else {
return "self.base.parse_atn_rule_with_precedence(atn(), rule_index, precedence)"
.to_owned();
}
if has_action_dispatch {
writeln!(
out,
"for action in actions {{ self.run_action(action, tree); }}"
)
.expect("writing to a string cannot fail");
} else {
writeln!(out, "let _ = actions;").expect("writing to a string cannot fail");
}
writeln!(out, "Ok(tree)").expect("writing to a string cannot fail");
out.lines()
.map(|line| format!(" {line}"))
.collect::<Vec<_>>()
.join("\n")
}
/// Renders a Rust parser module with one public method per grammar rule.
///
/// Parser methods use generated recursive-descent bodies for the ATN subset
/// covered by `parser_generated_rules` and keep the interpreter fallback for
/// unsupported constructs while the generated surface is expanded.
#[cfg(test)]
fn render_parser(grammar_name: &str, data: &CodegenData<'_>) -> io::Result<String> {
render_parser_with_options(grammar_name, data, ParserRenderOptions::default())
}
const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[
"reset",
"set_token_stream",
"token_stream",
"token_stream_mut",
"token_store",
"parse_tree_storage",
"clear_dfa",
"add_error_listener",
"remove_error_listeners",
"node",
"into_token_stream",
"into_token_store",
"into_parsed_file",
];
fn parser_public_rule_method_names(rule_names: &[String]) -> Vec<String> {
let mut used = GENERATED_PARSER_RESERVED_RULE_METHODS
.iter()
.map(|name| (*name).to_owned())
.collect::<BTreeSet<_>>();
rule_names
.iter()
.map(|rule| {
let base = rust_function_name(rule);
let name = unique_rule_method_name(&base, &used);
used.insert(name.clone());
name
})
.collect()
}
fn unique_rule_method_name(base: &str, used: &BTreeSet<String>) -> String {
if !used.contains(base) {
return base.to_owned();
}
let plain = base.strip_prefix("r#").unwrap_or(base);
let reserved_collision = GENERATED_PARSER_RESERVED_RULE_METHODS.contains(&base);
let stem = if reserved_collision {
format!("{plain}_rule")
} else {
plain.to_owned()
};
let (mut candidate, mut suffix) = if reserved_collision {
(stem.clone(), 2)
} else {
(format!("{stem}_2"), 3)
};
while used.contains(&candidate) {
candidate = format!("{stem}_{suffix}");
suffix += 1;
}
candidate
}
/// Everything embedded mode contributes to the rendered parser module.
#[derive(Debug, Default)]
struct EmbeddedParserData {
/// ATN action state -> translated inline Rust statements.
inline_actions: BTreeMap<usize, String>,
/// (rule, pred) -> (translated expr, optional `<fail=...>` message).
predicates: BTreeMap<(usize, usize), (String, Option<String>)>,
/// rule -> translated `@init` statements (run at rule entry).
init_entry: BTreeMap<usize, String>,
/// rule -> translated `@after` statements (run before `finish_rule`).
after: BTreeMap<usize, String>,
/// rule -> whether the rule declares args/returns/locals.
rule_has_attrs: Vec<bool>,
/// Rule-transition source state -> translated caller arg expression.
call_args: BTreeMap<usize, String>,
/// rule -> escaped name of its first declared arg, if any.
rule_arg0: Vec<Option<String>>,
/// Rendered `__RuleAttrsN` struct definitions.
attrs_structs: String,
/// Member fields lowered onto the parser struct.
struct_fields: String,
field_inits: String,
/// `@members` fn items + generated facades for the parser impl block.
impl_items: String,
/// `@members` structs/impls and generated support types.
module_items: String,
/// Decisions the ANTLR tool would compile to `adaptivePredict` calls
/// (non-LL(1) per [`tool_decision_analysis`]); the generated code must
/// route these through the simulator on every visit.
adaptive_decisions: BTreeSet<usize>,
/// Tool LOOK(1) dispatch intervals for LL(1)-disjoint decisions.
ll1_decision_arms: BTreeMap<usize, Vec<Vec<(i32, i32)>>>,
}
/// Raw grammar-local booleans whose actions and predicates are portable
/// without executing target-language code.
#[derive(Debug, Default)]
struct PortableLocalData {
/// Rule index -> generated local variable declarations.
declarations: Vec<Vec<String>>,
/// ATN action source state -> generated assignment.
inline_actions: BTreeMap<usize, String>,
/// Parser predicate coordinate -> generated boolean expression.
predicates: BTreeMap<(usize, usize), (String, Option<String>)>,
/// Rules whose local state has no equivalent interpreted representation.
required_generated_rules: BTreeSet<usize>,
}
impl PortableLocalData {
fn has_semantics(&self) -> bool {
!self.required_generated_rules.is_empty()
}
fn step_render(&self) -> Option<PortableLocalStepRender<'_>> {
self.has_semantics().then_some(PortableLocalStepRender {
declarations: &self.declarations,
inline_actions: &self.inline_actions,
predicates: &self.predicates,
required_generated_rules: &self.required_generated_rules,
})
}
}
fn portable_local_name(name: &str) -> String {
format!("__antlr_local_{name}")
}
fn parse_portable_bool_assignment(body: &str) -> Option<(&str, bool)> {
let body = body.trim();
let body = body.strip_suffix(';').unwrap_or(body);
let (target, value) = body.split_once('=')?;
let target = target.trim().strip_prefix('$')?;
let value = match value.trim() {
"true" => true,
"false" => false,
_ => return None,
};
(!target.is_empty()
&& target
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()))
.then_some((target, value))
}
fn parse_portable_bool_predicate(body: &str) -> Option<(&str, bool)> {
let body = body.trim();
let (body, negated) = body
.strip_prefix('!')
.map_or((body, false), |body| (body.trim(), true));
let name = body.strip_prefix('$')?.trim();
(!name.is_empty()
&& name
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()))
.then_some((name, negated))
}
fn build_structural_portable_local_data(
data: &CodegenData<'_>,
patterns: &SemPatternFile,
) -> io::Result<PortableLocalData> {
let model = structural_embedded_model(data, false)?;
let mut out = PortableLocalData {
declarations: vec![Vec::new(); data.rule_names.len()],
..PortableLocalData::default()
};
let mut local_names = Vec::with_capacity(data.rule_names.len());
for (rule_index, rule) in model.rules.iter().enumerate() {
let mut names = BTreeMap::new();
for attr in rule.attrs.iter().filter(|attr| attr.ty == "bool") {
let initial = if rule.local_names.contains(&attr.name) {
"false"
} else if rule.arg_names.first() == Some(&attr.name) {
"__precedence != 0"
} else {
continue;
};
let local = portable_local_name(&attr.name);
out.declarations[rule_index].push(format!("let mut {local} = {initial};"));
names.insert(attr.name.clone(), local);
}
local_names.push(names);
}
for action in structural_actions(data)? {
let Some((name, value)) = parse_portable_bool_assignment(&action.body) else {
continue;
};
let Some(local) = local_names
.get(action.rule_index)
.and_then(|names| names.get(name))
else {
continue;
};
if patterns
.coordinate_disposition(
SemanticsKind::ParserAction,
data.rule_names.get(action.rule_index).map(String::as_str),
None,
Some(action.state),
)
.is_some()
{
continue;
}
out.inline_actions
.insert(action.state, format!("{local} = {value};"));
out.required_generated_rules.insert(action.rule_index);
}
for predicate in structural_predicates(data)? {
let Some((name, negated)) = parse_portable_bool_predicate(&predicate.body) else {
continue;
};
let Some(local) = local_names
.get(predicate.rule_index)
.and_then(|names| names.get(name))
else {
continue;
};
if patterns
.coordinate_disposition(
SemanticsKind::ParserPredicate,
data.rule_names
.get(predicate.rule_index)
.map(String::as_str),
Some(predicate.predicate_index),
None,
)
.is_some()
{
continue;
}
let expression = if negated {
format!("!{local}")
} else {
local.clone()
};
out.predicates.insert(
(predicate.rule_index, predicate.predicate_index),
(expression, predicate.fail),
);
out.required_generated_rules.insert(predicate.rule_index);
}
Ok(out)
}
/// LOOK(1) of one decision alternative, plus whether the walk crossed a
/// predicate — Java nulls such alternatives, forcing the decision adaptive.
#[derive(Debug, Default)]
struct DecisionAltLook {
symbols: BTreeSet<i32>,
hit_pred: bool,
}
/// Ports the ANTLR tool's `AnalysisPipeline` classification: the decisions
/// whose alternatives' LOOK(1) sets are not pairwise disjoint (or hit a
/// predicate, or come up empty). Java compiles LL(1)-disjoint decisions to
/// plain token switches — no simulator, no learned DFA — and routes every
/// other decision through `adaptivePredict`, the only place DFA states are
/// learned and full-context diagnostics fire. `dumpDFA` and diagnostic
/// output therefore only match Java when the generated routing agrees.
fn tool_decision_analysis(data: &CodegenData<'_>) -> io::Result<ToolDecisionAnalysis> {
let atn = data.parser_atn()?;
let mut analysis = ToolDecisionAnalysis::default();
for (decision, state_number) in atn.decision_to_state().iter().enumerate() {
let Some(state) = atn.state(state_number) else {
continue;
};
// The tool never LL(1)-compiles non-greedy or left-recursion
// precedence decisions, disjoint LOOK or not — Java always emits
// `adaptivePredict` for them (a token switch would make a
// non-greedy loop greedy).
if state.non_greedy() || state.precedence_rule_decision() {
analysis.adaptive_decisions.insert(decision);
continue;
}
let looks: Vec<DecisionAltLook> = state
.transitions()
.iter()
.map(|transition| {
let mut look = DecisionAltLook::default();
let mut walk = DecisionLookWalk {
atn,
busy: BTreeSet::new(),
called_rules: vec![false; atn.rule_to_start_state().len()],
};
walk.walk(transition.target(), &mut Vec::new(), &mut look);
look
})
.collect();
if decision_alt_looks_disjoint(&looks) {
analysis.ll1_decision_arms.insert(
decision,
looks
.iter()
.map(|look| symbol_intervals(&look.symbols))
.collect(),
);
} else {
analysis.adaptive_decisions.insert(decision);
}
}
Ok(analysis)
}
/// Tool-side decision classification (see [`tool_decision_analysis`]).
#[derive(Debug, Default)]
struct ToolDecisionAnalysis {
/// Decisions Java compiles to `adaptivePredict` calls.
adaptive_decisions: BTreeSet<usize>,
/// Complete LOOK(1) dispatch intervals per alt for LL(1)-disjoint
/// decisions — exit alternatives included, unlike the within-rule
/// fast-path/LL(1) analyses, so legit input never falls through to the
/// simulator (Java's switch never does).
ll1_decision_arms: BTreeMap<usize, Vec<Vec<(i32, i32)>>>,
}
/// Collapses a sorted symbol set into inclusive intervals.
fn symbol_intervals(symbols: &BTreeSet<i32>) -> Vec<(i32, i32)> {
let mut intervals: Vec<(i32, i32)> = Vec::new();
for &symbol in symbols {
match intervals.last_mut() {
Some((_, stop)) if *stop + 1 == symbol => *stop = symbol,
_ => intervals.push((symbol, symbol)),
}
}
intervals
}
/// `AnalysisPipeline.disjoint`: pairwise-disjoint alt LOOK sets, with
/// Java's `getDecisionLookahead` nulling (empty or predicate-hitting sets)
/// folded in as an immediate non-LL(1) verdict.
fn decision_alt_looks_disjoint(looks: &[DecisionAltLook]) -> bool {
let mut combined = BTreeSet::new();
for look in looks {
if look.hit_pred || look.symbols.is_empty() {
return false;
}
if look.symbols.intersection(&combined).next().is_some() {
return false;
}
combined.extend(look.symbols.iter().copied());
}
true
}
struct DecisionLookWalk<'a> {
atn: &'a ParserAtn,
/// Java's `lookBusy`: (state, calling context) pairs already expanded.
busy: BTreeSet<(usize, Vec<usize>)>,
/// Java's `calledRuleStack`: rules on the walk's invocation path, the
/// recursion guard that bounds the context stack.
called_rules: Vec<bool>,
}
impl DecisionLookWalk<'_> {
/// `LL1Analyzer._LOOK` with an initially empty context: nested rule
/// invocations return precisely through `ctx`; the decision's own rule
/// boundary falls through the rule-stop state's return edges — the
/// deserializer materializes one per call site, which is exactly the
/// context-free FOLLOW Java walks there.
fn walk(&mut self, state_number: usize, ctx: &mut Vec<usize>, look: &mut DecisionAltLook) {
if !self.busy.insert((state_number, ctx.clone())) {
return;
}
let Some(state) = self.atn.state(state_number) else {
return;
};
if state.kind() == AtnStateKind::RuleStop {
if let Some(return_state) = ctx.pop() {
let cleared = state
.rule_index()
.map(|rule| std::mem::replace(&mut self.called_rules[rule], false));
self.walk(return_state, ctx, look);
if let (Some(rule), Some(flag)) = (state.rule_index(), cleared) {
self.called_rules[rule] = flag;
}
ctx.push(return_state);
return;
}
if state.transitions().is_empty() {
// The walk escaped through a stop state with no call sites —
// the start rule's end, where the only lookahead is EOF.
look.symbols.insert(TOKEN_EOF);
return;
}
}
for transition in state.transitions() {
match transition.data() {
ParserTransitionData::Rule {
target,
rule_index,
follow_state,
..
} => {
if self.called_rules.get(rule_index).copied().unwrap_or(true) {
continue;
}
self.called_rules[rule_index] = true;
ctx.push(follow_state);
self.walk(target, ctx, look);
ctx.pop();
self.called_rules[rule_index] = false;
}
ParserTransitionData::Predicate { .. }
| ParserTransitionData::Precedence { .. } => {
look.hit_pred = true;
}
ParserTransitionData::Epsilon { target }
| ParserTransitionData::Action { target, .. } => {
self.walk(target, ctx, look);
}
_ => {
// Terminal edge: enumerate the vocabulary through the
// shared `matches` so atoms, ranges, sets, negations and
// wildcards agree with the simulator exactly.
for symbol in TOKEN_EOF..=self.atn.max_token_type() {
if transition.matches(symbol, 1, self.atn.max_token_type()) {
look.symbols.insert(symbol);
}
}
}
}
}
}
}
/// Builds the embedded translation of every action, predicate, `@init` and
/// `@after` body in the rendered grammar, plus the members model.
fn build_embedded_parser_data(
data: &CodegenData<'_>,
type_name: &str,
grammar_name: &str,
options: ParserRenderOptions<'_>,
) -> io::Result<EmbeddedParserData> {
let model = structural_embedded_model(data, true)?;
let token_types: BTreeMap<String, i32> = data
.symbolic_names
.iter()
.enumerate()
.filter_map(|(token_type, name)| {
let name = name.as_ref()?;
i32::try_from(token_type)
.ok()
.map(|token_type| (name.clone(), token_type))
})
.collect();
let finish_body = |body: &str, translated: &str| -> String {
post_process_embedded(body, translated, type_name)
};
let decision_analysis = tool_decision_analysis(data)?;
let mut out = EmbeddedParserData {
rule_has_attrs: model
.rules
.iter()
.map(embedded::RuleModel::has_attrs)
.collect(),
adaptive_decisions: decision_analysis.adaptive_decisions,
ll1_decision_arms: decision_analysis.ll1_decision_arms,
..EmbeddedParserData::default()
};
for action in structural_actions(data)? {
if action.body.trim().is_empty() {
out.inline_actions.insert(action.state, String::new());
continue;
}
let ctx = embedded::TranslationCtx {
model: &model,
rule_index: action.rule_index,
body_offset: Some(
usize::try_from(action.span.bytes.start).expect("source offset exceeds usize"),
),
site: embedded::ActionSite::Body,
token_types: &token_types,
};
let translated = embedded::translate_body(&action.body, &ctx)?;
out.inline_actions
.insert(action.state, finish_body(&action.body, &translated));
}
for predicate in structural_predicates(data)? {
let ctx = embedded::TranslationCtx {
model: &model,
rule_index: predicate.rule_index,
body_offset: Some(
usize::try_from(predicate.span.bytes.start).expect("source offset exceeds usize"),
),
site: embedded::ActionSite::Body,
token_types: &token_types,
};
let translated = embedded::translate_body(predicate.body.trim(), &ctx)?;
out.predicates.insert(
(predicate.rule_index, predicate.predicate_index),
(
finish_body(&predicate.body, &translated),
predicate.fail.clone(),
),
);
}
// @init / @after bodies from the rule headers.
for (rule_index, rule) in model.rules.iter().enumerate() {
if let Some(body) = &rule.init_body {
let ctx = embedded::TranslationCtx {
model: &model,
rule_index,
body_offset: None,
site: embedded::ActionSite::Init,
token_types: &token_types,
};
let translated = embedded::translate_body(body, &ctx)?;
out.init_entry
.insert(rule_index, finish_body(body, &translated));
}
if let Some(body) = &rule.after_body {
let ctx = embedded::TranslationCtx {
model: &model,
rule_index,
body_offset: None,
site: embedded::ActionSite::After,
token_types: &token_types,
};
let translated = embedded::translate_body(body, &ctx)?;
out.after.insert(rule_index, finish_body(body, &translated));
}
}
// Per-rule attrs structs — emitted for every rule (context views
// reference them uniformly; attr-less rules get an empty struct).
for (rule_index, rule) in model.rules.iter().enumerate() {
let struct_name = embedded::attrs_struct_name(rule_index);
let mut fields = String::new();
for attr in &rule.attrs {
let _ = writeln!(
fields,
" pub {}: {},",
embedded::escape_keyword(&attr.name),
attr.ty
);
}
let _ = writeln!(
out.attrs_structs,
"#[derive(Clone, Debug, Default)]\n#[allow(non_snake_case, dead_code)]\npub struct {struct_name} {{\n{fields}}}\n"
);
}
// Members: fields, impl items, module items.
out.struct_fields
.push_str(" __embedded_pending_arg: Option<i64>,\n");
out.field_inits
.push_str(" __embedded_pending_arg: None,\n");
for field in &model.parser_members.fields {
let _ = writeln!(out.struct_fields, " {}: {},", field.name, field.ty);
let _ = writeln!(
out.field_inits,
" {}: {},",
field.name, field.init
);
}
for item in &model.parser_members.impl_items {
let item = post_process_embedded(item, item, type_name);
let mut indented = String::with_capacity(item.len());
for (line_index, line) in item.lines().enumerate() {
if line_index > 0 {
indented.push_str("\n ");
}
indented.push_str(line);
}
let _ = writeln!(out.impl_items, " {indented}\n");
}
for item in &model.parser_members.module_items {
let item = post_process_embedded(item, item, type_name);
let _ = writeln!(out.module_items, "{item}\n");
}
// Rule-call argument expressions attach to the exact finalized transition
// produced from each structural call element.
out.call_args = structural_embedded_rule_call_args(data)?;
out.rule_arg0 = model
.rules
.iter()
.map(|rule| {
rule.arg_names
.first()
.map(|name| embedded::escape_keyword(name))
})
.collect();
// Associated token constants: rendered bodies reference tokens as
// `TParser::NL` (post-processed to `Self::NL`).
for (name, token_type) in &token_types {
let _ = writeln!(
out.impl_items,
" #[allow(dead_code)]\n pub const {name}: i32 = {token_type};"
);
}
out.impl_items.push('\n');
// Recognizer-surface facades the rendered bodies call.
out.impl_items.push_str(&embedded_parser_facades());
out.module_items.push_str(EMBEDDED_INPUT_FACADE);
out.module_items.push_str(&render_embedded_context_types(
grammar_name,
data,
&model,
options,
));
Ok(out)
}
fn build_structural_parser_surface(
data: &CodegenData<'_>,
grammar_name: &str,
options: ParserRenderOptions<'_>,
) -> io::Result<EmbeddedParserData> {
let model = structural_embedded_model(data, false)?;
let mut out = EmbeddedParserData {
rule_has_attrs: model
.rules
.iter()
.map(embedded::RuleModel::has_attrs)
.collect(),
..EmbeddedParserData::default()
};
for (rule_index, rule) in model.rules.iter().enumerate() {
let struct_name = embedded::attrs_struct_name(rule_index);
let mut fields = String::new();
for attr in &rule.attrs {
let _ = writeln!(
fields,
" pub {}: {},",
embedded::escape_keyword(&attr.name),
attr.ty
);
}
let _ = writeln!(
out.attrs_structs,
"#[derive(Clone, Debug, Default)]\n#[allow(non_snake_case, dead_code)]\npub struct {struct_name} {{\n{fields}}}\n"
);
}
out.module_items.push_str(EMBEDDED_INPUT_FACADE);
out.module_items.push_str(&render_embedded_context_types(
grammar_name,
data,
&model,
options,
));
Ok(out)
}
/// Lowers argument text from each structural `rule[expr]` call onto that
/// element's finalized rule-transition state. Supports integer literals and
/// single identifiers (translated to the caller's `__attrs` field).
fn structural_embedded_rule_call_args(
data: &CodegenData<'_>,
) -> io::Result<BTreeMap<usize, String>> {
Ok(structural_rule_calls(data)?
.into_iter()
.filter_map(|call| {
let expression = embedded_rule_call_expression(call.arguments.as_deref()?)?;
Some((call.state, expression))
})
.collect())
}
fn embedded_rule_call_expression(value: &str) -> Option<String> {
let value = value.trim();
if value.parse::<i64>().is_ok() {
Some(value.to_owned())
} else if value
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
&& value
.chars()
.next()
.is_some_and(|ch| ch == '_' || ch.is_ascii_alphabetic())
{
// A bare identifier is a caller attribute (arg/local/return).
Some(format!("__attrs.{}", embedded::escape_keyword(value)))
} else {
None
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ContextSurfaceName {
context_type: String,
listener_method: String,
visitor_method: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ContextViewName {
surface: ContextSurfaceName,
rule_index: usize,
alternative_label: Option<String>,
}
#[derive(Debug, Eq, PartialEq)]
struct ContextSurfaceNames {
rules: Vec<ContextSurfaceName>,
views: Vec<ContextViewName>,
}
impl ContextSurfaceNames {
fn kind_id(&self, rule_index: usize, alternative_label: Option<&str>) -> usize {
self.views
.iter()
.position(|view| {
view.rule_index == rule_index
&& view.alternative_label.as_deref() == alternative_label
})
.expect("context view has an allocated dispatch identity")
}
}
/// Reserves canonical rule names before allocating labels. Alternative labels
/// always use a `Label`/`_label` suffix so their generated surfaces cannot be
/// confused with rule surfaces.
fn context_surface_names(model: &embedded::EmbeddedModel) -> ContextSurfaceNames {
let mut used_context_types = BTreeSet::from(["StoredTreeContext".to_owned()]);
let mut used_listener_methods = BTreeSet::from(["every_rule".to_owned()]);
let mut used_visitor_methods = BTreeSet::from([
"children".to_owned(),
"error_node".to_owned(),
"terminal".to_owned(),
]);
let rules = model
.rules
.iter()
.map(|rule| ContextSurfaceName {
context_type: allocate_rule_context_type(&rule.name, &mut used_context_types),
listener_method: allocate_rule_listener_method(&rule.name, &mut used_listener_methods),
visitor_method: allocate_rule_listener_method(&rule.name, &mut used_visitor_methods),
})
.collect::<Vec<_>>();
let mut alternatives = (0..model.rules.len())
.map(|_| BTreeMap::new())
.collect::<Vec<_>>();
let mut views = Vec::new();
for (rule_index, rule) in model.rules.iter().enumerate() {
views.push(ContextViewName {
surface: rules[rule_index].clone(),
rule_index,
alternative_label: None,
});
for alternative in &rule.alts {
let Some(label) = &alternative.label else {
continue;
};
if let Entry::Vacant(entry) = alternatives[rule_index].entry(label.clone()) {
let surface = ContextSurfaceName {
context_type: allocate_label_context_type(label, &mut used_context_types),
listener_method: allocate_label_listener_method(
label,
&mut used_listener_methods,
),
visitor_method: allocate_label_listener_method(
label,
&mut used_visitor_methods,
),
};
entry.insert(surface.clone());
views.push(ContextViewName {
surface,
rule_index,
alternative_label: Some(label.clone()),
});
}
}
}
ContextSurfaceNames { rules, views }
}
fn allocate_rule_context_type(source_name: &str, used: &mut BTreeSet<String>) -> String {
let base = rust_type_name(source_name);
let canonical = format!("{base}Context");
if used.insert(canonical.clone()) {
return canonical;
}
allocate_numbered_context_type(&format!("{base}Rule"), used)
}
fn allocate_label_context_type(source_name: &str, used: &mut BTreeSet<String>) -> String {
allocate_numbered_context_type(&format!("{}Label", rust_type_name(source_name)), used)
}
fn allocate_numbered_context_type(stem: &str, used: &mut BTreeSet<String>) -> String {
let mut candidate = format!("{stem}Context");
let mut suffix = 2;
while !used.insert(candidate.clone()) {
candidate = format!("{stem}{suffix}Context");
suffix += 1;
}
candidate
}
fn allocate_rule_listener_method(source_name: &str, used: &mut BTreeSet<String>) -> String {
let canonical = rust_function_name(source_name)
.trim_start_matches("r#")
.to_owned();
if used.insert(canonical.clone()) {
return canonical;
}
allocate_numbered_listener_method(&format!("{canonical}_rule"), used)
}
fn allocate_label_listener_method(source_name: &str, used: &mut BTreeSet<String>) -> String {
let canonical = rust_function_name(source_name);
let stem = format!("{}_label", canonical.trim_start_matches("r#"));
allocate_numbered_listener_method(&stem, used)
}
fn allocate_numbered_listener_method(stem: &str, used: &mut BTreeSet<String>) -> String {
let mut candidate = stem.to_owned();
let mut suffix = 2;
while !used.insert(candidate.clone()) {
candidate = format!("{stem}_{suffix}");
suffix += 1;
}
candidate
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ContextAlternativeDispatch {
runtime_alt_number: usize,
kind_id: usize,
operator: bool,
}
fn context_alternative_dispatch(
rule_index: usize,
rule: &embedded::RuleModel,
names: &ContextSurfaceNames,
) -> (bool, Vec<ContextAlternativeDispatch>) {
let left_recursive = rule
.alts
.iter()
.any(|alternative| alternative.is_lr_operator(&rule.name));
let mut primary_alt_number = 0;
let mut operator_alt_number = 0;
let alternatives = rule
.alts
.iter()
.enumerate()
.map(|(authored_alt_index, alternative)| {
let operator = left_recursive && alternative.is_lr_operator(&rule.name);
let runtime_alt_number = if operator {
operator_alt_number += 1;
operator_alt_number
} else if left_recursive {
primary_alt_number += 1;
primary_alt_number
} else {
authored_alt_index + 1
};
ContextAlternativeDispatch {
runtime_alt_number,
kind_id: names.kind_id(rule_index, alternative.label.as_deref()),
operator,
}
})
.collect();
(left_recursive, alternatives)
}
fn render_context_alt_kind_match(
alternatives: &[ContextAlternativeDispatch],
fallback_kind: usize,
alt_number: &str,
) -> String {
if alternatives.is_empty()
|| alternatives
.iter()
.all(|alternative| alternative.kind_id == fallback_kind)
{
return fallback_kind.to_string();
}
let mut arms = String::new();
for alternative in alternatives {
let _ = writeln!(
arms,
" {} => {},",
alternative.runtime_alt_number, alternative.kind_id
);
}
let distinct_kinds = alternatives
.iter()
.map(|alternative| alternative.kind_id)
.collect::<BTreeSet<_>>();
if distinct_kinds.len() == 1 {
let only_kind = distinct_kinds
.first()
.copied()
.expect("non-empty alternatives have one context kind");
let _ = writeln!(arms, " 0 => {only_kind},");
}
format!(
"match {alt_number} {{\n{arms} _ => {fallback_kind},\n }}"
)
}
fn render_context_kind_functions(
model: &embedded::EmbeddedModel,
names: &ContextSurfaceNames,
) -> String {
if names
.views
.iter()
.all(|view| view.alternative_label.is_none())
{
return r#"#[allow(dead_code)]
fn __context_kind(context: RuleNodeView<'_>) -> usize {
context.rule_index()
}
#[allow(dead_code)]
fn __active_context_kind(
context: &antlr4_runtime::ParserRuleContext,
_storage: &antlr4_runtime::ParseTreeStorage,
_tokens: &antlr4_runtime::TokenStore,
) -> usize {
context.rule_index()
}
"#
.to_owned();
}
let mut stored_arms = String::new();
let mut active_arms = String::new();
for (rule_index, rule) in model.rules.iter().enumerate() {
let fallback_kind = names.kind_id(rule_index, None);
let (left_recursive, alternatives) = context_alternative_dispatch(rule_index, rule, names);
if !left_recursive {
let matcher = render_context_alt_kind_match(
&alternatives,
fallback_kind,
"context.context_alt_number()",
);
let _ = writeln!(
stored_arms,
" {rule_index} => {{\n {matcher}\n }},"
);
let _ = writeln!(
active_arms,
" {rule_index} => {{\n {matcher}\n }},"
);
continue;
}
let primary = alternatives
.iter()
.copied()
.filter(|alternative| !alternative.operator)
.collect::<Vec<_>>();
let operators = alternatives
.iter()
.copied()
.filter(|alternative| alternative.operator)
.collect::<Vec<_>>();
let primary_match =
render_context_alt_kind_match(&primary, fallback_kind, "context.context_alt_number()");
let operator_match = render_context_alt_kind_match(
&operators,
fallback_kind,
"context.context_alt_number()",
);
let _ = writeln!(
stored_arms,
" {rule_index} => {{\n let operator = context.children().next().and_then(antlr4_runtime::Node::as_rule).is_some_and(|child| child.rule_index() == {rule_index});\n if operator {{\n {operator_match}\n }} else {{\n {primary_match}\n }}\n }},"
);
let _ = writeln!(
active_arms,
" {rule_index} => {{\n let operator = context.child_nodes(storage, tokens).next().and_then(antlr4_runtime::Node::as_rule).is_some_and(|child| child.rule_index() == {rule_index});\n if operator {{\n {operator_match}\n }} else {{\n {primary_match}\n }}\n }},"
);
}
format!(
r#"#[allow(dead_code)]
fn __context_kind(context: RuleNodeView<'_>) -> usize {{
match context.rule_index() {{
{stored_arms} _ => usize::MAX,
}}
}}
#[allow(dead_code)]
fn __active_context_kind(
context: &antlr4_runtime::ParserRuleContext,
storage: &antlr4_runtime::ParseTreeStorage,
tokens: &antlr4_runtime::TokenStore,
) -> usize {{
match context.rule_index() {{
{active_arms} _ => usize::MAX,
}}
}}
"#
)
}
fn context_alternatives<'a>(
rule: &'a embedded::RuleModel,
alternative_label: Option<&str>,
) -> Vec<&'a embedded::AltModel> {
rule.alts
.iter()
.filter(|alternative| {
alternative_label.is_none_or(|label| alternative.label.as_deref() == Some(label))
})
.collect()
}
fn context_child_cardinalities(
rule: &embedded::RuleModel,
alternative_label: Option<&str>,
) -> BTreeMap<String, embedded::ChildCardinality> {
choice_child_cardinalities(
context_alternatives(rule, alternative_label)
.into_iter()
.map(|alternative| alternative.children.clone()),
)
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ContextLabelAccessor {
source_name: String,
target: String,
token_types: Vec<i32>,
cardinality: embedded::ChildCardinality,
selector: ContextLabelSelector,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ContextLabelSelector {
Nth(usize),
LastAfter(usize),
AllAfter(usize),
}
fn context_label_accessors(
rule: &embedded::RuleModel,
alternative_label: Option<&str>,
) -> Vec<ContextLabelAccessor> {
let alternatives = context_alternatives(rule, alternative_label);
let labels = alternatives
.iter()
.flat_map(|alternative| {
alternative
.refs
.iter()
.filter_map(|element| element.label.clone())
})
.collect::<BTreeSet<_>>();
labels
.into_iter()
.filter_map(|label| context_label_accessor(&alternatives, label))
.collect()
}
fn context_label_accessor(
alternatives: &[&embedded::AltModel],
label: String,
) -> Option<ContextLabelAccessor> {
let declarations = alternatives
.iter()
.flat_map(|alternative| alternative.refs.iter())
.filter(|element| element.label.as_deref() == Some(label.as_str()))
.collect::<Vec<_>>();
let first = declarations.first()?;
if (first.target.is_empty() && first.token_types.is_empty())
|| declarations.iter().any(|element| {
!element.stable_accessor
|| !same_context_ref_target(element, first)
|| element.is_list != first.is_list
})
{
return None;
}
let target = first.target.clone();
let token_types = first.token_types.clone();
let is_list = first.is_list;
let mut selector = None;
let mut cardinalities = Vec::with_capacity(alternatives.len());
for alternative in alternatives {
let matching = alternative
.refs
.iter()
.enumerate()
.filter(|(_, element)| element.label.as_deref() == Some(label.as_str()))
.collect::<Vec<_>>();
if matching.is_empty() {
let target_cardinality = sum_child_cardinalities(
alternative
.refs
.iter()
.filter(|element| context_ref_can_match_target(element, first))
.map(|element| element.cardinality),
);
if target_cardinality.max != Some(0) {
return None;
}
cardinalities.push(embedded::ChildCardinality::ZERO);
continue;
}
let alternative_selector =
context_label_selector(alternative, &matching, &label, first, is_list)?;
if selector.is_some_and(|existing| existing != alternative_selector) {
return None;
}
selector = Some(alternative_selector);
cardinalities.push(if is_list {
sum_child_cardinalities(matching.iter().map(|(_, element)| element.cardinality))
} else {
matching[0].1.cardinality
});
}
let mut cardinality = choice_cardinality(&cardinalities);
if !is_list {
cardinality = embedded::ChildCardinality {
min: usize::from(cardinality.min > 0),
max: Some(usize::from(cardinality.max != Some(0))),
};
}
Some(ContextLabelAccessor {
source_name: label,
target,
token_types,
cardinality,
selector: selector?,
})
}
fn context_label_selector(
alternative: &embedded::AltModel,
matching: &[(usize, &embedded::ElementRef)],
label: &str,
target: &embedded::ElementRef,
is_list: bool,
) -> Option<ContextLabelSelector> {
let first_position = matching[0].0;
let start = exact_target_cardinality(&alternative.refs[..first_position], target)?;
if is_list {
let has_unlabeled_target = alternative.refs[first_position..].iter().any(|element| {
context_ref_can_match_target(element, target)
&& element.cardinality.max != Some(0)
&& element.label.as_deref() != Some(label)
});
return (!has_unlabeled_target).then_some(ContextLabelSelector::AllAfter(start));
}
if matching.len() != 1 {
return None;
}
let element = matching[0].1;
if !element.cardinality.is_repeated() {
return Some(ContextLabelSelector::Nth(start));
}
let has_following_target = alternative.refs[first_position + 1..]
.iter()
.any(|following| {
context_ref_can_match_target(following, target) && following.cardinality.max != Some(0)
});
(!has_following_target).then_some(ContextLabelSelector::LastAfter(start))
}
fn same_context_ref_target(left: &embedded::ElementRef, right: &embedded::ElementRef) -> bool {
if left.token_types.is_empty() && right.token_types.is_empty() {
left.target == right.target
} else {
left.token_types == right.token_types
}
}
fn context_ref_can_match_target(
element: &embedded::ElementRef,
target: &embedded::ElementRef,
) -> bool {
if element.token_types.is_empty() || target.token_types.is_empty() {
return same_context_ref_target(element, target);
}
element
.token_types
.iter()
.any(|token_type| target.token_types.contains(token_type))
}
fn exact_target_cardinality(
refs: &[embedded::ElementRef],
target: &embedded::ElementRef,
) -> Option<usize> {
refs.iter().try_fold(0_usize, |total, element| {
if !context_ref_can_match_target(element, target) {
return Some(total);
}
if !element.token_types.is_empty()
&& !element
.token_types
.iter()
.all(|token_type| target.token_types.contains(token_type))
{
return None;
}
let exact = element.cardinality.max?;
(element.cardinality.min == exact).then(|| total.saturating_add(exact))
})
}
fn sum_child_cardinalities(
cardinalities: impl IntoIterator<Item = embedded::ChildCardinality>,
) -> embedded::ChildCardinality {
cardinalities.into_iter().fold(
embedded::ChildCardinality::ZERO,
|mut total, cardinality| {
total.min = total.min.saturating_add(cardinality.min);
total.max = match (total.max, cardinality.max) {
(Some(current), Some(next)) => Some(current.saturating_add(next)),
_ => None,
};
total
},
)
}
fn choice_cardinality(alternatives: &[embedded::ChildCardinality]) -> embedded::ChildCardinality {
let mut min = usize::MAX;
let mut max = Some(0_usize);
for cardinality in alternatives {
min = min.min(cardinality.min);
max = match (max, cardinality.max) {
(Some(current), Some(next)) => Some(current.max(next)),
_ => None,
};
}
embedded::ChildCardinality {
min: if min == usize::MAX { 0 } else { min },
max,
}
}
fn allocate_context_method(
preferred: String,
fallback_stem: &str,
used: &mut BTreeSet<String>,
) -> String {
if used.insert(preferred.clone()) {
return preferred;
}
allocate_numbered_listener_method(fallback_stem, used)
}
fn accessor_stem(name: &str) -> String {
rust_function_name(name).trim_start_matches("r#").to_owned()
}
fn render_rule_label_accessor(
out: &mut String,
method: &str,
view_name: &str,
child_view: &str,
child_index: usize,
label: &ContextLabelAccessor,
) {
if let ContextLabelSelector::AllAfter(skip) = label.selector {
let _ = writeln!(
out,
" pub fn {method}(&self) -> impl Iterator<Item = {child_view}<'a>> + '_ {{\n __rule_children(self.__node, {child_index})\n .skip({skip})\n .map(move |node| {child_view}::__from_child_node(node, &self.__invocation_states))\n }}"
);
return;
}
let lookup = match label.selector {
ContextLabelSelector::Nth(occurrence) => format!(".nth({occurrence})"),
ContextLabelSelector::LastAfter(skip) => format!(".skip({skip}).last()"),
ContextLabelSelector::AllAfter(_) => unreachable!("handled above"),
};
if label.cardinality.is_required_single() {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Result<{child_view}<'a>, MissingChildError> {{\n __rule_children(self.__node, {child_index})\n {lookup}\n .map(|node| {child_view}::__from_child_node(node, &self.__invocation_states))\n .ok_or_else(|| MissingChildError::new(\"{view_name}\", \"{}\"))\n }}",
label.source_name
);
} else {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Option<{child_view}<'a>> {{\n __rule_children(self.__node, {child_index})\n {lookup}\n .map(|node| {child_view}::__from_child_node(node, &self.__invocation_states))\n }}"
);
}
}
fn render_token_label_accessor(
out: &mut String,
method: &str,
view_name: &str,
label: &ContextLabelAccessor,
) {
let token_types = label
.token_types
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(", ");
let children = if let [token_type] = label.token_types.as_slice() {
format!("__token_children(self.__node, {token_type})")
} else {
format!("__token_children_matching(self.__node, &[{token_types}])")
};
if let ContextLabelSelector::AllAfter(skip) = label.selector {
let _ = writeln!(
out,
" pub fn {method}(&self) -> impl Iterator<Item = TerminalNode<'a>> + '_ {{\n {children}\n .skip({skip})\n .map(TerminalNode::new)\n }}"
);
return;
}
let lookup = match label.selector {
ContextLabelSelector::Nth(occurrence) => format!(".nth({occurrence})"),
ContextLabelSelector::LastAfter(skip) => format!(".skip({skip}).last()"),
ContextLabelSelector::AllAfter(_) => unreachable!("handled above"),
};
if label.cardinality.is_required_single() {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Result<TerminalNode<'a>, MissingChildError> {{\n {children}\n {lookup}\n .map(TerminalNode::new)\n .ok_or_else(|| MissingChildError::new(\"{view_name}\", \"{}\"))\n }}",
label.source_name
);
} else {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Option<TerminalNode<'a>> {{\n {children}\n {lookup}\n .map(TerminalNode::new)\n }}"
);
}
}
fn render_context_child_accessors(
view_name: &str,
model: &embedded::EmbeddedModel,
context_names: &ContextSurfaceNames,
token_accessors: &[(String, i32)],
child_cardinalities: &BTreeMap<String, embedded::ChildCardinality>,
label_accessors: &[ContextLabelAccessor],
) -> String {
let mut out = String::new();
let _ = writeln!(
out,
" pub fn child_count(&self) -> usize {{\n match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_count(),\n __GeneratedRuleContext::Active {{ context, .. }} => context.child_count(),\n }}\n }}\n\n pub fn start(&self) -> __GeneratedTokenView {{\n let token = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.start(),\n __GeneratedRuleContext::Active {{ context, tokens, .. }} => context.start(tokens),\n }};\n __GeneratedTokenView {{ text: token.map(|token| token.text_or_empty().to_owned()).unwrap_or_default() }}\n }}"
);
let mut used_methods = BTreeSet::from([
"child_count".to_owned(),
"rule_node".to_owned(),
"start".to_owned(),
]);
for (child_index, child) in model
.rules
.iter()
.enumerate()
.filter(|(_, child)| child_cardinalities.contains_key(child.name.as_str()))
{
let cardinality = child_cardinalities[child.name.as_str()];
let stem = accessor_stem(&child.name);
let preferred = if cardinality.is_repeated() {
format!("{stem}_children")
} else {
rust_function_name(&child.name)
};
let method =
allocate_context_method(preferred, &format!("{stem}_rule_child"), &mut used_methods);
let child_view = &context_names.rules[child_index].context_type;
if cardinality.is_repeated() {
let _ = writeln!(
out,
" pub fn {method}(&self) -> impl Iterator<Item = {child_view}<'a>> + '_ {{\n __rule_children(self.__node, {child_index})\n .map(move |node| {child_view}::__from_child_node(node, &self.__invocation_states))\n }}"
);
} else if cardinality.is_required_single() {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Result<{child_view}<'a>, MissingChildError> {{\n __rule_children(self.__node, {child_index})\n .next()\n .map(|node| {child_view}::__from_child_node(node, &self.__invocation_states))\n .ok_or_else(|| MissingChildError::new(\"{view_name}\", \"{}\"))\n }}",
child.name
);
} else {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Option<{child_view}<'a>> {{\n __rule_children(self.__node, {child_index})\n .next()\n .map(|node| {child_view}::__from_child_node(node, &self.__invocation_states))\n }}"
);
}
}
for (token_name, token_type) in token_accessors
.iter()
.filter(|(token_name, _)| child_cardinalities.contains_key(token_name.as_str()))
{
let cardinality = child_cardinalities[token_name.as_str()];
let stem = accessor_stem(token_name);
let preferred = if cardinality.is_repeated() {
format!("{stem}_tokens")
} else {
format!("{stem}_token")
};
let method = allocate_context_method(
preferred,
&format!("{stem}_terminal_child"),
&mut used_methods,
);
if cardinality.is_repeated() {
let _ = writeln!(
out,
" pub fn {method}(&self) -> impl Iterator<Item = TerminalNode<'a>> + '_ {{\n __token_children(self.__node, {token_type}).map(TerminalNode::new)\n }}"
);
} else if cardinality.is_required_single() {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Result<TerminalNode<'a>, MissingChildError> {{\n __token_children(self.__node, {token_type})\n .next()\n .map(TerminalNode::new)\n .ok_or_else(|| MissingChildError::new(\"{view_name}\", \"{token_name}\"))\n }}"
);
} else {
let _ = writeln!(
out,
" pub fn {method}(&self) -> Option<TerminalNode<'a>> {{\n __token_children(self.__node, {token_type})\n .next()\n .map(TerminalNode::new)\n }}"
);
}
}
for label in label_accessors {
let stem = accessor_stem(&label.source_name);
let method = allocate_context_method(
rust_function_name(&label.source_name),
&format!("{stem}_label"),
&mut used_methods,
);
if label.token_types.is_empty()
&& let Some(child_index) = model
.rules
.iter()
.position(|child| child.name == label.target)
{
let child_view = &context_names.rules[child_index].context_type;
render_rule_label_accessor(
&mut out,
&method,
view_name,
child_view,
child_index,
label,
);
continue;
}
if label.token_types.is_empty() {
continue;
}
render_token_label_accessor(&mut out, &method, view_name, label);
}
out
}
/// Generates the typed context views, listener trait, and walker for the
/// embedded `.test.stg` surface:
///
/// * one `<Rule>Context` view per parser rule (plus one per labeled
/// alternative) with cardinality-aware rule, token, and label accessors,
/// `child_count()`, `start()`, public attribute fields, and a `FromRuleNode`
/// impl backing `ctx.downcast_ref::<XContext>()`;
/// * the `<Grammar>Listener` trait with defaulted `enter_/exit_<rule>` (and
/// per-labeled-alternative) callbacks plus terminal/error-node visitors;
/// * a module-local `ParseTreeWalker` whose bridge dispatches the runtime
/// walker onto the typed listener callbacks, threading the invoking-state
/// chain Java's `RuleContext.toString` renders (`[13 6]`).
fn render_embedded_context_types(
grammar_name: &str,
data: &CodegenData<'_>,
model: &embedded::EmbeddedModel,
options: ParserRenderOptions<'_>,
) -> String {
let mut out = String::new();
let context_names = context_surface_names(model);
let surface_name = grammar_name.strip_suffix("Parser").unwrap_or(grammar_name);
let listener_trait = format!("{surface_name}Listener");
let visitor_trait = format!("{surface_name}Visitor");
let visitable_trait = format!("{surface_name}Visitable");
let tree_walker = format!("{surface_name}TreeWalker");
let token_accessors = std::iter::once(("EOF".to_owned(), TOKEN_EOF))
.chain(
data.symbolic_names
.iter()
.enumerate()
.filter_map(|(token_type, name)| {
let name = name.as_ref()?;
i32::try_from(token_type).ok().map(|ty| (name.clone(), ty))
}),
)
.collect::<Vec<_>>();
if options.generate_visitor {
let _ = writeln!(
out,
"#[allow(dead_code)]\npub trait {visitable_trait}<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a>;\n}}\n\nimpl<'a> {visitable_trait}<'a> for antlr4_runtime::Node<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self }}\n}}\n\nimpl<'a> {visitable_trait}<'a> for RuleNodeView<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.node() }}\n}}\n"
);
}
out.push_str(
r#"#[allow(dead_code)]
#[derive(Clone)]
pub struct TerminalNode<'a> {
__node: RuntimeTerminalNode<'a>,
}
#[allow(dead_code)]
impl<'a> TerminalNode<'a> {
fn new(node: RuntimeTerminalNode<'a>) -> Self {
Self { __node: node }
}
pub fn symbol(&self) -> antlr4_runtime::TokenView<'a> {
self.__node.symbol()
}
}
impl std::fmt::Display for TerminalNode<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.__node.text())
}
}
#[allow(dead_code)]
#[derive(Clone)]
pub struct ErrorNode<'a> {
__node: RuntimeErrorNode<'a>,
}
#[allow(dead_code)]
impl<'a> ErrorNode<'a> {
fn new(node: RuntimeErrorNode<'a>) -> Self {
Self { __node: node }
}
pub fn symbol(&self) -> antlr4_runtime::TokenView<'a> {
self.__node.symbol()
}
}
impl std::fmt::Display for ErrorNode<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.__node.text())
}
}
#[allow(dead_code)]
#[derive(Clone, Copy)]
enum __GeneratedRuleContext<'a> {
Stored(RuleNodeView<'a>),
Active {
context: &'a antlr4_runtime::ParserRuleContext,
storage: &'a antlr4_runtime::ParseTreeStorage,
tokens: &'a antlr4_runtime::TokenStore,
},
}
#[doc(hidden)]
#[derive(Clone, Copy, Debug)]
pub struct StoredTreeContext;
#[derive(Clone, Copy, Debug)]
struct __ActiveParserContext;
#[allow(dead_code)]
fn __context_children<'a>(
source: __GeneratedRuleContext<'a>,
) -> impl Iterator<Item = antlr4_runtime::Node<'a>> + 'a {
let mut stored = match source {
__GeneratedRuleContext::Stored(node) => Some(node.children()),
__GeneratedRuleContext::Active { .. } => None,
};
let mut active = match source {
__GeneratedRuleContext::Stored(_) => None,
__GeneratedRuleContext::Active {
context,
storage,
tokens,
} => Some(context.child_nodes(storage, tokens)),
};
std::iter::from_fn(move || {
stored
.as_mut()
.and_then(Iterator::next)
.or_else(|| active.as_mut().and_then(Iterator::next))
})
}
#[allow(dead_code)]
fn __rule_children<'a>(
source: __GeneratedRuleContext<'a>,
rule_index: usize,
) -> impl Iterator<Item = RuleNodeView<'a>> + 'a {
__context_children(source).filter_map(move |child| {
let rule = child.as_rule()?;
(rule.rule_index() == rule_index).then_some(rule)
})
}
#[allow(dead_code)]
fn __token_children<'a>(
source: __GeneratedRuleContext<'a>,
token_type: i32,
) -> impl Iterator<Item = RuntimeTerminalNode<'a>> + 'a {
__context_children(source).filter_map(move |child| {
let terminal = match child.kind() {
antlr4_runtime::NodeKind::Terminal => child.as_terminal(),
antlr4_runtime::NodeKind::Error => {
child.as_error().map(antlr4_runtime::ErrorNodeView::terminal)
}
antlr4_runtime::NodeKind::Rule => None,
}?;
(terminal.symbol().token_type() == token_type).then_some(terminal)
})
}
#[allow(dead_code)]
fn __token_children_matching<'a>(
source: __GeneratedRuleContext<'a>,
token_types: &'static [i32],
) -> impl Iterator<Item = RuntimeTerminalNode<'a>> + 'a {
__context_children(source).filter_map(move |child| {
let terminal = match child.kind() {
antlr4_runtime::NodeKind::Terminal => child.as_terminal(),
antlr4_runtime::NodeKind::Error => {
child.as_error().map(antlr4_runtime::ErrorNodeView::terminal)
}
antlr4_runtime::NodeKind::Rule => None,
}?;
token_types
.contains(&terminal.symbol().token_type())
.then_some(terminal)
})
}
#[allow(dead_code)]
trait __FromActiveRuleContext<'a>: Sized {
fn __from_active(
context: &'a antlr4_runtime::ParserRuleContext,
invocation_states: Vec<isize>,
storage: &'a antlr4_runtime::ParseTreeStorage,
tokens: &'a antlr4_runtime::TokenStore,
) -> Option<Self>;
}
#[allow(dead_code)]
fn __active_context_view<'a, T: __FromActiveRuleContext<'a>>(
context: &'a antlr4_runtime::ParserRuleContext,
invocation_states: Vec<isize>,
storage: &'a antlr4_runtime::ParseTreeStorage,
tokens: &'a antlr4_runtime::TokenStore,
) -> Option<T> {
T::__from_active(context, invocation_states, storage, tokens)
}
"#,
);
if options.generate_visitor {
let _ = writeln!(
out,
"impl<'a> {visitable_trait}<'a> for TerminalNode<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.__node.node() }}\n}}\n\nimpl<'a> {visitable_trait}<'a> for &TerminalNode<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.__node.node() }}\n}}\n\nimpl<'a> {visitable_trait}<'a> for ErrorNode<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.__node.node() }}\n}}\n\nimpl<'a> {visitable_trait}<'a> for &ErrorNode<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.__node.node() }}\n}}\n"
);
}
out.push_str(&render_context_kind_functions(model, &context_names));
for ContextViewName {
surface,
rule_index,
alternative_label,
} in &context_names.views
{
let view_name = &surface.context_type;
let rule = &model.rules[*rule_index];
let context_kind = context_names.kind_id(*rule_index, alternative_label.as_deref());
let stored_kind_guard = alternative_label.as_ref().map_or(String::new(), |_| {
format!(" || __context_kind(node) != {context_kind}")
});
let active_kind_guard = alternative_label.as_ref().map_or(String::new(), |_| {
format!(" || __active_context_kind(context, storage, tokens) != {context_kind}")
});
let child_cardinalities = context_child_cardinalities(rule, alternative_label.as_deref());
let label_accessors = context_label_accessors(rule, alternative_label.as_deref());
let attrs_struct = embedded::attrs_struct_name(*rule_index);
let mut fields = String::new();
let mut field_inits = String::new();
for attr in &rule.attrs {
let name = embedded::escape_keyword(&attr.name);
let _ = writeln!(fields, " pub {name}: {ty},", ty = attr.ty);
let _ = writeln!(field_inits, " {name}: __attrs.{name}.clone(),");
}
let _ = writeln!(
out,
"#[allow(non_camel_case_types, dead_code)]\n#[derive(Clone)]\npub struct {view_name}<'a, State = StoredTreeContext> {{\n __node: __GeneratedRuleContext<'a>,\n __invocation_states: Vec<isize>,\n __state: std::marker::PhantomData<State>,\n{fields}}}\n"
);
let _ = writeln!(
out,
"impl<'a> FromRuleNode<'a> for {view_name}<'a> {{\n fn from_rule_node(node: RuleNodeView<'a>) -> Option<Self> {{\n if node.rule_index() != {rule_index}{stored_kind_guard} {{ return None; }}\n Some(Self::__from_node(node))\n }}\n}}\n\nimpl<'a> AsRuleNode<'a> for {view_name}<'a> {{\n fn as_rule_node(&self) -> RuleNodeView<'a> {{ self.rule_node() }}\n}}\n\nimpl<'a> {view_name}<'a> {{\n pub fn rule_node(&self) -> RuleNodeView<'a> {{\n match self.__node {{\n __GeneratedRuleContext::Stored(node) => node,\n __GeneratedRuleContext::Active {{ .. }} => unreachable!(\"stored context type contains an active parser context\"),\n }}\n }}\n}}\n\nimpl<'a> __FromActiveRuleContext<'a> for {view_name}<'a, __ActiveParserContext> {{\n fn __from_active(\n context: &'a antlr4_runtime::ParserRuleContext,\n invocation_states: Vec<isize>,\n storage: &'a antlr4_runtime::ParseTreeStorage,\n tokens: &'a antlr4_runtime::TokenStore,\n ) -> Option<Self> {{\n if context.rule_index() != {rule_index}{active_kind_guard} {{ return None; }}\n let __default = {attrs_struct}::default();\n let __attrs = context.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Some(Self {{\n __node: __GeneratedRuleContext::Active {{ context, storage, tokens }},\n __invocation_states: invocation_states,\n __state: std::marker::PhantomData,\n{field_inits} }})\n }}\n}}\n"
);
let mut accessors = String::new();
let _ = writeln!(
accessors,
" fn __from_node(node: RuleNodeView<'a>) -> Self {{\n let invocation_states = node.invocation_states().collect();\n Self::__from_node_with_invocation_states(node, invocation_states)\n }}\n\n fn __from_child_node(node: RuleNodeView<'a>, parent_invocation_states: &[isize]) -> Self {{\n let mut invocation_states = Vec::with_capacity(parent_invocation_states.len() + 1);\n invocation_states.push(node.invoking_state());\n invocation_states.extend_from_slice(parent_invocation_states);\n Self::__from_node_with_invocation_states(node, invocation_states)\n }}\n\n fn __from_listener_node(node: RuleNodeView<'a>, invocation_states: Option<&[isize]>) -> Self {{\n invocation_states.map_or_else(\n || Self::__from_node(node),\n |states| Self::__from_node_with_invocation_states(node, states.to_vec()),\n )\n }}\n\n fn __from_node_with_invocation_states(node: RuleNodeView<'a>, invocation_states: Vec<isize>) -> Self {{\n let __default = {attrs_struct}::default();\n let __attrs = node.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Self {{\n __node: __GeneratedRuleContext::Stored(node),\n __invocation_states: invocation_states,\n __state: std::marker::PhantomData,\n{field_inits} }}\n }}\n"
);
let common_accessors = render_context_child_accessors(
view_name,
model,
&context_names,
&token_accessors,
&child_cardinalities,
&label_accessors,
);
let _ = writeln!(
out,
"#[allow(dead_code, clippy::all)]\nimpl<'a> {view_name}<'a> {{\n{accessors}}}\n\n#[allow(dead_code, clippy::all)]\nimpl<'a, State> {view_name}<'a, State> {{\n{common_accessors}}}\n"
);
if options.generate_visitor {
let _ = writeln!(
out,
"impl<'a> {visitable_trait}<'a> for {view_name}<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.rule_node().node() }}\n}}\n\nimpl<'a> {visitable_trait}<'a> for &{view_name}<'a> {{\n fn into_parse_tree_node(self) -> antlr4_runtime::Node<'a> {{ self.rule_node().node() }}\n}}\n"
);
}
// Java's RuleContext.toString(): bracketed invoking-state chain from
// this context to the root, the root's sentinel excluded.
let _ = writeln!(
out,
"impl<State> std::fmt::Display for {view_name}<'_, State> {{\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{\n let chain: Vec<String> = self.__invocation_states.iter().map(|state| state.to_string()).collect();\n write!(f, \"[{{}}]\", chain.join(\" \"))\n }}\n}}\n"
);
}
if options.generate_listener {
let mut trait_methods = String::new();
let mut enter_arms = String::new();
let mut exit_arms = String::new();
for (kind_id, view) in context_names.views.iter().enumerate() {
let ContextSurfaceName {
context_type,
listener_method,
..
} = &view.surface;
let _ = writeln!(
trait_methods,
" fn enter_{listener_method}(&mut self, _ctx: &{context_type}) -> Result<(), E> {{ Ok(()) }}\n fn exit_{listener_method}(&mut self, _ctx: &{context_type}) -> Result<(), E> {{ Ok(()) }}"
);
let _ = writeln!(
enter_arms,
" {kind_id} => listener.enter_{listener_method}(&{context_type}::__from_listener_node(context, invocation_states.as_deref()))?,"
);
let _ = writeln!(
exit_arms,
" {kind_id} => listener.exit_{listener_method}(&{context_type}::__from_listener_node(context, invocation_states.as_deref()))?,"
);
}
let _ = writeln!(
out,
"#[allow(dead_code, unused_variables)]\npub trait {listener_trait}<E = std::convert::Infallible> {{\n fn walk(&mut self, tree: antlr4_runtime::Node<'_>) -> Result<(), E>\n where\n Self: Sized,\n {{\n {tree_walker}::walk(self, tree)\n }}\n\n fn enter_every_rule(&mut self, _ctx: RuleNodeView<'_>) -> Result<(), E> {{ Ok(()) }}\n fn exit_every_rule(&mut self, _ctx: RuleNodeView<'_>) -> Result<(), E> {{ Ok(()) }}\n\n{trait_methods} fn visit_terminal(&mut self, _node: &TerminalNode) -> Result<(), E> {{ Ok(()) }}\n fn visit_error_node(&mut self, _node: &ErrorNode) -> Result<(), E> {{ Ok(()) }}\n fn output(&mut self) -> std::io::Stdout {{ std::io::stdout() }}\n}}\n"
);
let _ = writeln!(
out,
r#"#[allow(dead_code)]
pub struct {tree_walker};
#[allow(dead_code)]
impl {tree_walker} {{
pub fn walk<E, T: {listener_trait}<E>>(
listener: &mut T,
tree: antlr4_runtime::Node<'_>,
) -> Result<(), E> {{
Self::__walk(listener, tree, None)
}}
pub fn walk_with_invocation_states<E, T: {listener_trait}<E>>(
listener: &mut T,
tree: antlr4_runtime::Node<'_>,
parent_invocation_states: Vec<isize>,
) -> Result<(), E> {{
Self::__walk(listener, tree, Some(parent_invocation_states))
}}
fn __walk<E, T: {listener_trait}<E>>(
listener: &mut T,
tree: antlr4_runtime::Node<'_>,
mut invocation_states: Option<Vec<isize>>,
) -> Result<(), E> {{
enum Event<'tree> {{
Enter(antlr4_runtime::Node<'tree>),
Exit(RuleNodeView<'tree>),
}}
let mut stack = vec![Event::Enter(tree)];
while let Some(event) = stack.pop() {{
match event {{
Event::Enter(node) => match node.kind() {{
antlr4_runtime::NodeKind::Rule => {{
let context = node.as_rule().expect("rule node kind checked");
if let Some(states) = &mut invocation_states {{
states.insert(0, context.invoking_state());
}}
listener.enter_every_rule(context)?;
match __context_kind(context) {{
{enter_arms} _ => {{}}
}}
stack.push(Event::Exit(context));
stack.extend(context.children().rev().map(Event::Enter));
}}
antlr4_runtime::NodeKind::Terminal => {{
listener.visit_terminal(&TerminalNode::new(
node.as_terminal().expect("terminal node kind checked"),
))?;
}}
antlr4_runtime::NodeKind::Error => {{
listener.visit_error_node(&ErrorNode::new(
node.as_error().expect("error node kind checked"),
))?;
}}
}},
Event::Exit(context) => {{
match __context_kind(context) {{
{exit_arms} _ => {{}}
}}
listener.exit_every_rule(context)?;
if let Some(states) = &mut invocation_states {{
states.remove(0);
}}
}}
}}
}}
Ok(())
}}
}}
pub type ParseTreeWalker = {tree_walker};
"#
);
}
if options.generate_visitor {
let mut visitor_methods = String::new();
let mut visitor_arms = String::new();
for (kind_id, view) in context_names.views.iter().enumerate() {
let ContextSurfaceName {
context_type,
visitor_method,
..
} = &view.surface;
let _ = writeln!(
visitor_methods,
" fn visit_{visitor_method}(&mut self, ctx: &{context_type}) -> Self::Result {{\n self.visit_children(ctx)\n }}"
);
let _ = writeln!(
visitor_arms,
" {kind_id} => {visitor_trait}::visit_{visitor_method}(self.0, &{context_type}::__from_listener_node(context, None)),"
);
}
let _ = writeln!(
out,
r#"#[allow(dead_code, unused_variables)]
pub trait {visitor_trait}: Sized {{
type Result;
fn default_result(&mut self) -> Self::Result;
fn visit<'tree, T>(&mut self, tree: T) -> Self::Result
where
T: {visitable_trait}<'tree>,
{{
let tree = {visitable_trait}::into_parse_tree_node(tree);
let mut bridge = __VisitorBridge(self);
antlr4_runtime::ParseTreeVisitor::visit(&mut bridge, tree)
}}
fn visit_children<'tree, T>(&mut self, context: T) -> Self::Result
where
T: {visitable_trait}<'tree>,
{{
let tree = {visitable_trait}::into_parse_tree_node(context);
let context = tree.as_rule().expect("visit_children requires a rule context");
let mut bridge = __VisitorBridge(self);
antlr4_runtime::ParseTreeVisitor::visit_children(&mut bridge, context)
}}
fn aggregate_result(
&mut self,
_aggregate: Self::Result,
next_result: Self::Result,
) -> Self::Result {{
next_result
}}
fn should_visit_next_child(
&mut self,
_context: RuleNodeView<'_>,
_current_result: &Self::Result,
) -> bool {{
true
}}
fn visit_terminal(&mut self, _node: &TerminalNode) -> Self::Result {{
self.default_result()
}}
fn visit_error_node(&mut self, _node: &ErrorNode) -> Self::Result {{
self.default_result()
}}
{visitor_methods}}}
#[allow(dead_code)]
struct __VisitorBridge<'a, T: {visitor_trait}>(&'a mut T);
impl<T: {visitor_trait}> antlr4_runtime::ParseTreeVisitor for __VisitorBridge<'_, T> {{
type Result = T::Result;
fn visit_rule(&mut self, context: RuleNodeView<'_>) -> Self::Result {{
match __context_kind(context) {{
{visitor_arms} _ => {visitor_trait}::default_result(self.0),
}}
}}
fn visit_terminal(&mut self, node: RuntimeTerminalNode<'_>) -> Self::Result {{
{visitor_trait}::visit_terminal(self.0, &TerminalNode::new(node))
}}
fn visit_error_node(&mut self, node: RuntimeErrorNode<'_>) -> Self::Result {{
{visitor_trait}::visit_error_node(self.0, &ErrorNode::new(node))
}}
fn default_result(&mut self) -> Self::Result {{
{visitor_trait}::default_result(self.0)
}}
fn aggregate_result(
&mut self,
aggregate: Self::Result,
next_result: Self::Result,
) -> Self::Result {{
{visitor_trait}::aggregate_result(self.0, aggregate, next_result)
}}
fn should_visit_next_child(
&mut self,
context: RuleNodeView<'_>,
current_result: &Self::Result,
) -> bool {{
{visitor_trait}::should_visit_next_child(self.0, context, current_result)
}}
}}
"#
);
}
out
}
/// Post-translation cleanups shared by all embedded bodies: `TParser::NL`
/// becomes `Self::NL` so token constants resolve inside the generic impl.
fn post_process_embedded(_original: &str, translated: &str, type_name: &str) -> String {
let translated = replace_all(translated, &format!("{type_name}::"), "Self::");
let translated = replace_all(
&translated,
"(&__ctx).to_string_tree(Some(self), self.base.token_store())",
"(&__ctx).to_string_tree(Some(self), self.base.parse_tree_storage(), self.base.token_store())",
);
replace_all(
&translated,
"(&__ctx).to_string_tree(Some(self))",
"(&__ctx).to_string_tree(Some(self), self.base.parse_tree_storage(), self.base.token_store())",
)
}
/// Plain substring replacement (`str::replace` is a disallowed method here).
fn replace_all(text: &str, needle: &str, replacement: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(index) = rest.find(needle) {
out.push_str(&rest[..index]);
out.push_str(replacement);
rest = &rest[index + needle.len()..];
}
out.push_str(rest);
out
}
/// Generated-recognizer surface used by rendered test actions.
fn embedded_parser_facades() -> String {
r#" #[allow(dead_code)]
fn output(&mut self) -> std::io::Stdout {
std::io::stdout()
}
#[allow(dead_code)]
fn input(&mut self) -> __GeneratedInput<'_, L> {
__GeneratedInput(self.base.input())
}
#[allow(dead_code)]
fn interpreter(&mut self) -> &mut Self {
self
}
#[allow(dead_code)]
fn set_error_handler(&mut self, _strategy: antlr4_runtime::BailErrorStrategy) {
self.base.set_bail_on_error(true);
}
#[allow(dead_code)]
fn expected_tokens(&mut self) -> antlr4_runtime::ExpectedTokenSet {
self.base.expected_tokens_current(atn())
}
#[allow(dead_code)]
fn rule_invocation_stack(&self) -> Vec<String> {
self.base.rule_invocation_stack()
}
#[allow(dead_code)]
fn dump_dfa(&mut self) {
if let Some(simulator) = self.simulator.as_ref() {
print!("{}", simulator.dump_dfa_java_style(self.base.vocabulary()));
}
}
"#
.to_owned()
}
/// Token-stream facade matching the `.test.stg` surface
/// (`self.input().text()` / `.la(i)` / `.lt(i).text()`).
const EMBEDDED_INPUT_FACADE: &str = r#"
#[allow(dead_code)]
pub struct __GeneratedInput<'a, L: TokenSource>(&'a mut CommonTokenStream<L>);
#[allow(dead_code)]
impl<L: TokenSource> __GeneratedInput<'_, L> {
pub fn text(&mut self) -> String {
self.0.text_all()
}
pub fn la(&mut self, offset: isize) -> i32 {
antlr4_runtime::IntStream::la(self.0, offset)
}
pub fn lt(&mut self, offset: isize) -> __GeneratedTokenView {
__GeneratedTokenView {
text: self
.0
.lt(offset)
.map(|token| token.text_or_empty().to_owned())
.unwrap_or_default(),
}
}
}
#[allow(dead_code)]
pub struct __GeneratedTokenView {
text: String,
}
#[allow(dead_code)]
impl __GeneratedTokenView {
pub fn text(&self) -> &str {
&self.text
}
}
"#;
/// Collects `assume-*`-overridden action states that should become explicit
/// empty `run_action` arms instead of falling through to user hooks.
fn collect_noop_action_states(
data: &CodegenData<'_>,
patterns: &SemPatternFile,
) -> io::Result<BTreeSet<usize>> {
let mut noop_action_states = BTreeSet::new();
let action_state_rules = parser_action_state_rules(data)?;
for state in action_state_rules.keys() {
if parser_action_assume_overridden(patterns, data, &action_state_rules, *state) {
noop_action_states.insert(*state);
}
}
Ok(noop_action_states)
}
/// Public per-rule entry methods (`pub fn s(&mut self) -> …`).
fn render_public_rule_methods(public_rule_method_names: &[String]) -> String {
let mut rule_methods = String::new();
for (index, rule_method_name) in public_rule_method_names.iter().enumerate() {
writeln!(
rule_methods,
" pub fn {rule_method_name}(&mut self) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
writeln!(rule_methods, " self.parse_rule({index})")
.expect("writing to a string cannot fail");
writeln!(rule_methods, " }}").expect("writing to a string cannot fail");
}
rule_methods
}
/// Step-render view over the embedded data.
///
/// `force_adaptive` stays off: per-decision routing follows the tool
/// classification in `adaptive_decisions` instead, matching which decisions
/// Java compiles to switches versus `adaptivePredict` calls.
fn embedded_step_render(embedded: &EmbeddedParserData) -> EmbeddedStepRender<'_> {
EmbeddedStepRender {
force_adaptive: false,
adaptive_decisions: &embedded.adaptive_decisions,
ll1_decision_arms: &embedded.ll1_decision_arms,
predicates: &embedded.predicates,
rule_has_attrs: &embedded.rule_has_attrs,
init_entry: &embedded.init_entry,
after: &embedded.after,
call_args: &embedded.call_args,
rule_arg0: &embedded.rule_arg0,
}
}
/// The module/struct/impl text an [`EmbeddedParserData`] contributes to the
/// rendered parser, empty in template mode.
fn embedded_render_slots(
embedded_data: Option<&EmbeddedParserData>,
) -> (String, String, String, String, String) {
embedded_data.map_or_else(
|| {
(
String::new(),
String::new(),
String::new(),
String::new(),
String::new(),
)
},
|embedded| {
(
embedded.attrs_structs.clone(),
embedded.module_items.clone(),
embedded.struct_fields.clone(),
embedded.field_inits.clone(),
embedded.impl_items.clone(),
)
},
)
}
fn render_parser_with_options(
grammar_name: &str,
data: &CodegenData<'_>,
options: ParserRenderOptions<'_>,
) -> io::Result<String> {
let empty_patterns = SemPatternFile::default();
let patterns = options.patterns.unwrap_or(&empty_patterns);
let type_name = rust_type_name(grammar_name);
let metadata = render_parser_metadata(grammar_name, data);
let parser_atn = data.parser_atn()?;
let parser_atn_data = render_u32_slice(parser_atn.packed_words());
let token_constants = render_token_constants(data);
let rule_constants = render_rule_constants(data);
// Embedded mode: the grammar carries real Rust action/predicate bodies
// (rendered through the target `.test.stg`); translate and splice them
// instead of recognizing template markup.
let embedded_data = if options.embedded {
Some(build_embedded_parser_data(
data,
&type_name,
grammar_name,
options,
)?)
} else {
None
};
let structural_surface = if options.embedded {
None
} else {
Some(build_structural_parser_surface(
data,
grammar_name,
options,
)?)
};
let embedded_step_render = embedded_data.as_ref().map(embedded_step_render);
let mut portable_local_data = if options.embedded {
PortableLocalData::default()
} else {
build_structural_portable_local_data(data, patterns)?
};
portable_local_data.required_generated_rules =
parser_rule_callers_reaching(data, &portable_local_data.required_generated_rules)?;
// A per-coordinate `assume-*` override is a documented no-op fallback; it
// must not fall through to `parser_action_hook`, which would fail loud under
// the Error policy or run a user side effect for a coordinate the manifest
// reports as ignored. A `hook`/`error`
// override falls through to the `parser_action_hook` catch-all, but an
// `assume-*` override gets an explicit empty arm.
let mut noop_action_states = collect_noop_action_states(data, patterns)?;
// ANTLR-synthesized action states (left-recursion elimination, etc.) are
// no-ops with no author intent. They must NOT reach `parser_action_hook`
// either, or they would fail loud at runtime under the Error policy — the
// same treatment `enforce_sem_unknown` gives them at codegen time. Give each
// an explicit empty `run_action` arm.
noop_action_states.extend(synthetic_parser_action_states(data)?);
// Authored empty action bodies are explicit no-ops too: they carry source
// provenance for the manifest but should not disable generated parsing or
// fall through to runtime hooks under strict semantic policies.
noop_action_states.extend(empty_parser_action_states(data)?);
let predicates = if options.embedded {
Vec::new()
} else {
structural_predicate_templates(data, SemanticsKind::ParserPredicate, patterns)?
};
let rule_args = if options.embedded {
Vec::new()
} else {
structural_parser_rule_args(data)?
};
let inline_action_statements = embedded_data.as_ref().map_or_else(
|| portable_local_data.inline_actions.clone(),
|embedded| embedded.inline_actions.clone(),
);
let inline_action_states = inline_action_statements
.keys()
.copied()
.collect::<BTreeSet<_>>();
let action_states = parser_action_states(data)?
.into_iter()
.collect::<BTreeSet<_>>();
let mut generated_action_states = if options.embedded {
action_states.clone()
} else {
// Synthetic actions and explicit assume-* overrides are no-op states.
// They should not disable generated parser rules just because they have
// an ATN action transition; real author actions still fall through to
// the interpreted path in non-embedded mode.
noop_action_states
.intersection(&action_states)
.copied()
.collect()
};
generated_action_states.extend(portable_local_data.inline_actions.keys().copied());
// Under a non-default unknown-coordinate policy every predicate transition
// must reach the interpreter, which applies the policy to the complete
// structurally bound coordinate inventory.
let predicate_coordinates = parser_predicate_transitions(data)?
.into_iter()
.collect::<BTreeSet<_>>();
let mut generated_predicate_coordinates = if options.embedded {
predicate_coordinates.clone()
} else {
predicates
.iter()
.filter_map(|(coordinate, predicate)| {
can_generate_parser_predicate(predicate).then_some(*coordinate)
})
.collect::<BTreeSet<_>>()
};
generated_predicate_coordinates.extend(portable_local_data.predicates.keys().copied());
let has_action_dispatch = !action_states.is_empty();
let has_predicate_dispatch = !predicates.is_empty();
let track_alt_numbers = uses_alt_number_contexts(data);
let track_context_alt_numbers = uses_structural_context_alt_numbers(data)?;
let generated_rule_enabled = vec![true; data.rule_names.len()];
let generated_rules = parser_generated_rules(
data,
&generated_rule_enabled,
&rule_args,
ActionStateSets {
all: &action_states,
generated: &generated_action_states,
inline: &inline_action_states,
},
PredicateCoordinateSets {
all: &predicate_coordinates,
generated: &generated_predicate_coordinates,
},
(has_action_dispatch || has_predicate_dispatch || portable_local_data.has_semantics())
&& !options.embedded,
)?;
require_portable_local_rules_generated(
&generated_rules,
&portable_local_data.required_generated_rules,
data,
)?;
if options.require_generated_parser || options.embedded {
// Embedded actions only run on the generated path, so every rule must
// compile; an interpreted fallback would silently skip them.
require_all_parser_rules_generated(&generated_rules, data)?;
}
let direct_generated_rule_calls = generated_rules
.iter()
.map(Option::is_some)
.collect::<Vec<_>>();
let generated_rule_dispatch = render_generated_rule_dispatch_with_rule_names(
&generated_rules,
&direct_generated_rule_calls,
&data.rule_names,
&inline_action_statements,
track_alt_numbers,
track_context_alt_numbers,
embedded_step_render,
portable_local_data.step_render(),
);
// A non-default policy must reach the interpreter through the emitted
// runtime options, so its literal forces the options-carrying call shape.
//
// A `hook`-disposed coordinate falls through to the configured policy when
// its hook is unimplemented (the documented `SemIR → hook → policy` chain),
// so it does NOT escalate the global policy: doing so would flip unrelated
// `assume-true` coordinates in the same grammar to fail-loud. Users select
// fail-loud for missing hooks with `--sem-unknown=error` /
// `--require-full-semantics`; under the default policy a declined hook keeps
// historical pass-through, per coordinate.
let unknown_policy_literal = match options.sem_unknown {
SemUnknownPolicy::AssumeTrue => None,
SemUnknownPolicy::AssumeFalse => Some("antlr4_runtime::UnknownSemanticPolicy::AssumeFalse"),
SemUnknownPolicy::Hook | SemUnknownPolicy::Error => {
Some("antlr4_runtime::UnknownSemanticPolicy::Error")
}
};
let parse_rule_fallback = render_parser_parse_rule_fallback(
track_alt_numbers,
track_context_alt_numbers,
&rule_args,
has_action_dispatch,
has_predicate_dispatch,
unknown_policy_literal,
);
let parser_semantics_function = render_parser_semantics_function(&predicates, data)?;
let typed_hook_adapter =
render_typed_hook_adapter(&type_name, &parser_typed_hook_mappings(data, patterns)?);
let typed_parser_constructor = if typed_hook_adapter.is_empty() {
String::new()
} else {
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
format!(
"impl<L, T> {type_name}<L, {adapter_name}<T>>\nwhere\n L: TokenSource,\n T: {trait_name},\n{{\n pub fn with_typed_hooks(input: CommonTokenStream<L>, hooks: T) -> Self {{\n Self::with_hooks(input, {adapter_name}::new(hooks))\n }}\n}}\n"
)
};
// The adaptive-direct path runs `parse_atn_rule_adaptive_or_fallback`, which
// falls back through `parse_atn_rule` without the `ParserRuntimeOptions`
// emitted below. A non-default unknown-predicate policy only reaches the
// interpreter through those options, so it must not take that shortcut, or
// an untranslated predicate would silently pass instead of applying the
// configured fail/assume-false behavior.
let adaptive_direct_allowed = !has_action_dispatch
&& !track_alt_numbers
&& !track_context_alt_numbers
&& !has_predicate_dispatch
&& unknown_policy_literal.is_none();
let embedded_noop_states = BTreeSet::new();
let action_method = render_parser_action_method(
!action_states.is_empty() && !options.embedded,
if options.embedded {
&embedded_noop_states
} else {
&noop_action_states
},
);
let parse_convenience = render_parser_parse_convenience(&type_name);
let base_initialization = render_parser_base_initialization(unknown_policy_literal);
let public_rule_method_names = parser_public_rule_method_names(&data.rule_names);
let entry_rule_indices = likely_parser_entry_rule_indices(data)?;
let parser_rustdoc = render_parser_rustdoc(&public_rule_method_names, &entry_rule_indices);
let rule_methods = render_public_rule_methods(&public_rule_method_names);
let (
embedded_attrs_structs,
embedded_module_items,
embedded_struct_fields,
embedded_field_inits,
embedded_impl_items,
) = embedded_render_slots(embedded_data.as_ref().or(structural_surface.as_ref()));
let generated_header = GENERATED_MODULE_HEADER;
let generated_footer = GENERATED_MODULE_FOOTER;
let embedded_imports = if embedded_data.is_some() || structural_surface.is_some() {
"#[allow(unused_imports)]\nuse std::io::Write as _;\n#[allow(unused_imports)]\nuse antlr4_runtime::{java_style_list, PredictionMode, BailErrorStrategy, TerminalNodeView as RuntimeTerminalNode, ErrorNodeView as RuntimeErrorNode, RuleNodeView, AsRuleNode, FromRuleNode, MissingChildError, Token as _};\n"
} else {
""
};
Ok(format!(
r#"{generated_header}use antlr4_runtime::recognizer::RecognizerData;
use antlr4_runtime::token::TokenSource;
use antlr4_runtime::token_stream::CommonTokenStream;
use antlr4_runtime::atn::parser_atn::ParserAtn;
use antlr4_runtime::{{BaseParser, GeneratedParser, GrammarMetadata, Parser, Recognizer}};
use std::sync::OnceLock;
{embedded_imports}
{token_constants}
{rule_constants}
{metadata}
{parser_semantics_function}
{typed_hook_adapter}
{embedded_attrs_structs}
{embedded_module_items}
static PARSER_ATN_DATA: &[u32] = &{parser_atn_data};
static ATN_CELL: OnceLock<ParserAtn> = OnceLock::new();
/// Validates and caches the packed grammar ATN for all parser instances.
fn atn() -> &'static ParserAtn {{
ATN_CELL.get_or_init(|| {{
ParserAtn::from_static(PARSER_ATN_DATA)
.unwrap_or_else(|error| panic!("generated parser ATN is incompatible with this runtime: {{error}}"))
}})
}}
/// Borrows the validated packed parser ATN embedded in this module.
pub fn parser_atn() -> &'static ParserAtn {{
atn()
}}
{parse_convenience}
{parser_rustdoc}#[derive(Debug)]
pub struct {type_name}<L, H = antlr4_runtime::NoSemanticHooks>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
base: BaseParser<L, H>,
simulator: Option<antlr4_runtime::ParserAtnSimulator<'static>>,
generated_only: bool,
{embedded_struct_fields}}}
#[allow(dead_code)]
#[derive(Debug)]
enum GeneratedRuleError {{
Fatal(antlr4_runtime::AntlrError),
}}
impl GeneratedRuleError {{
fn into_error(self) -> antlr4_runtime::AntlrError {{
match self {{
Self::Fatal(error) => error,
}}
}}
}}
impl<L> {type_name}<L, antlr4_runtime::NoSemanticHooks>
where
L: TokenSource,
{{
pub fn new(input: CommonTokenStream<L>) -> Self {{
Self::with_hooks(input, antlr4_runtime::NoSemanticHooks)
}}
}}
impl<L, H> {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
pub fn with_hooks(input: CommonTokenStream<L>, hooks: H) -> Self {{
let grammar_metadata = metadata();
let data = RecognizerData::new(
grammar_metadata.grammar_file_name(),
grammar_metadata.vocabulary(),
)
.with_rule_names(grammar_metadata.rule_names().iter().copied())
.with_channel_names(grammar_metadata.channel_names().iter().copied())
.with_mode_names(grammar_metadata.mode_names().iter().copied());
{base_initialization}
Self {{
base,
simulator: None,
generated_only: std::env::var_os("ANTLR4_RUST_GENERATED_ONLY").is_some(),
{embedded_field_inits} }}
}}
pub fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
/// Adds a listener for parser diagnostics.
pub fn add_error_listener<T>(&mut self, listener: T)
where
T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,
{{
self.base.add_error_listener(listener);
}}
/// Removes every parser error listener, including the default console listener.
pub fn remove_error_listeners(&mut self) {{
self.base.remove_error_listeners();
}}
/// Fully resets parser-owned state and rewinds the current token stream.
pub fn reset(&mut self) {{
self.base.reset();
if let Some(simulator) = self.simulator.as_mut() {{
simulator.reset();
}}
}}
/// Replaces the token stream and fully resets parser-owned state.
pub fn set_token_stream(&mut self, input: CommonTokenStream<L>) {{
self.base.set_token_stream(input);
if let Some(simulator) = self.simulator.as_mut() {{
simulator.reset();
}}
}}
#[must_use]
pub const fn token_stream(&self) -> &CommonTokenStream<L> {{
self.base.token_stream()
}}
#[must_use]
pub const fn token_stream_mut(&mut self) -> &mut CommonTokenStream<L> {{
self.base.token_stream_mut()
}}
#[must_use]
pub const fn token_store(&self) -> &antlr4_runtime::TokenStore {{
self.base.token_store()
}}
#[must_use]
pub const fn parse_tree_storage(&self) -> &antlr4_runtime::ParseTreeStorage {{
self.base.parse_tree_storage()
}}
#[must_use]
pub fn prediction_context_stats(&self) -> antlr4_runtime::PredictionContextStats {{
self.simulator.as_ref().map_or_else(
antlr4_runtime::PredictionContextStats::default,
antlr4_runtime::ParserAtnSimulator::prediction_context_stats,
)
}}
#[must_use]
pub fn parser_dfa_stats(&self) -> antlr4_runtime::ParserDfaStats {{
self.simulator.as_ref().map_or_else(
antlr4_runtime::ParserDfaStats::default,
antlr4_runtime::ParserAtnSimulator::parser_dfa_stats,
)
}}
/// Clears this grammar's learned parser decision DFAs.
pub fn clear_dfa(&mut self) {{
if let Some(simulator) = self.simulator.as_mut() {{
simulator.clear_dfa();
}} else {{
antlr4_runtime::ParserAtnSimulator::clear_shared_dfa(atn());
}}
}}
#[must_use]
pub fn node(&self, id: antlr4_runtime::NodeId) -> antlr4_runtime::Node<'_> {{
self.base.node(id)
}}
#[must_use]
pub fn into_token_stream(self) -> CommonTokenStream<L> {{
self.base.into_token_stream()
}}
#[must_use]
pub fn into_token_store(self) -> antlr4_runtime::TokenStore {{
self.base.into_token_store()
}}
#[must_use]
pub fn into_parsed_file(self, root: antlr4_runtime::NodeId) -> antlr4_runtime::ParsedFile {{
self.base.into_parsed_file(root)
}}
#[allow(dead_code)]
fn simulator(&mut self) -> &mut antlr4_runtime::ParserAtnSimulator<'static> {{
self.simulator
.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()))
}}
#[allow(dead_code)]
fn generated_only(&self) -> bool {{
self.generated_only
}}
#[allow(dead_code)]
fn parse_rule(&mut self, rule_index: usize) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_rule_precedence(rule_index, 0)
}}
#[allow(dead_code)]
fn parse_rule_precedence(&mut self, rule_index: usize, precedence: i32) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_rule_precedence_inner(rule_index, precedence, true)
}}
#[allow(dead_code)]
fn parse_rule_precedence_from_generated(&mut self, rule_index: usize, precedence: i32) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_rule_precedence_inner(rule_index, precedence, false)
}}
#[allow(dead_code)]
fn parse_rule_precedence_inner(&mut self, rule_index: usize, precedence: i32, allow_generated_fallback: bool) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
if allow_generated_fallback {{
// True top-level entry: drop any fail-loud coordinates left by a
// previous parse so a reused parser starts clean. Mid-parse the hits
// are preserved so a generated parent can surface a recovered child's
// fail-loud coordinate at this boundary.
self.base.reset_unknown_semantic_hits();
}}
let __rule_start = antlr4_runtime::IntStream::index(self.base.input());
let __generated_only = self.generated_only();
let __tree = if let Some(result) = self.parse_generated_rule(rule_index, precedence, allow_generated_fallback) {{
match result {{
Ok(tree) => tree,
Err(error) => {{
antlr4_runtime::IntStream::seek(self.base.input(), __rule_start);
// A generated predicate that consulted an unimplemented hook
// (returning None under the Error policy) fails the alternative
// and surfaces here as a generic failed-predicate/rule error.
// The documented contract is to fail loud with
// `AntlrError::Unsupported`, so prefer a recorded semantic error
// over the generic one — but only at the top-level entry, mirroring
// the post-parse check below: a nested child keeps its hits so the
// generated parent surfaces them at that boundary instead.
if allow_generated_fallback {{
if let Some(semantic_error) = self.base.take_unknown_semantic_error() {{
return Err(semantic_error);
}}
}}
return Err(error.into_error());
}}
}}
}} else if __generated_only {{
return Err(antlr4_runtime::AntlrError::Unsupported(format!("generated parser did not emit rule {{}}", rule_index)));
}} else {{
self.parse_interpreted_rule_precedence(rule_index, precedence)?
}};
// Surface unknown-predicate coordinates recorded under the Error policy
// at the top-level entry. Generated predicate steps evaluate on the
// committed path and are recovered as rule errors, so a parse that
// consulted an unimplemented hook predicate must fail with
// `AntlrError::Unsupported` instead of returning a recovered `Ok` tree.
if allow_generated_fallback {{
if let Some(error) = self.base.take_unknown_semantic_error() {{
return Err(error);
}}
}}
if allow_generated_fallback {{
self.base.report_generated_parser_diagnostics();
if let Some(error) = self.base.take_unknown_semantic_error() {{
return Err(error);
}}
}}
Ok(__tree)
}}
#[allow(dead_code)]
fn parse_interpreted_rule(&mut self, rule_index: usize) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_interpreted_rule_precedence(rule_index, 0)
}}
#[allow(dead_code)]
fn parse_interpreted_rule_precedence(&mut self, rule_index: usize, precedence: i32) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
if precedence == 0 && {adaptive_direct_allowed} && std::env::var_os("ANTLR4_RUST_ADAPTIVE_DIRECT").is_some() {{
let simulator = self
.simulator
.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));
self.base
.parse_atn_rule_adaptive_or_fallback(atn(), simulator, rule_index)
}} else {{
{parse_rule_fallback}
}}
}}
{generated_rule_dispatch}
{embedded_impl_items}
{rule_methods}
{action_method}
}}
{typed_parser_constructor}
impl<L, H> GeneratedParser for {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
fn parser_atn() -> &'static ParserAtn {{
parser_atn()
}}
}}
impl<L, H> Recognizer for {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
fn data(&self) -> &antlr4_runtime::RecognizerData {{
self.base.data()
}}
fn data_mut(&mut self) -> &mut antlr4_runtime::RecognizerData {{
self.base.data_mut()
}}
}}
impl<L, H> Parser for {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
fn build_parse_trees(&self) -> bool {{ self.base.build_parse_trees() }}
fn set_build_parse_trees(&mut self, build: bool) {{ self.base.set_build_parse_trees(build); }}
fn number_of_syntax_errors(&self) -> usize {{ self.base.number_of_syntax_errors() }}
fn report_diagnostic_errors(&self) -> bool {{ self.base.report_diagnostic_errors() }}
fn set_report_diagnostic_errors(&mut self, report: bool) {{ self.base.set_report_diagnostic_errors(report); }}
fn prediction_mode(&self) -> antlr4_runtime::PredictionMode {{ self.base.prediction_mode() }}
fn set_prediction_mode(&mut self, mode: antlr4_runtime::PredictionMode) {{ self.base.set_prediction_mode(mode); }}
}}
{generated_footer}"#
))
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum ActionTemplate {
LexerPopMode,
Hook(SemanticHelperCall),
UnsupportedLexerAction { rule_name: String, body: String },
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum PredicateTemplate {
Hook,
True,
False,
FalseWithMessage {
message: String,
},
/// A non-constant-false predicate carrying an ANTLR `<fail=...>` message.
/// Transparent to evaluation — `inner` provides the truth value and codegen;
/// the message is surfaced only when `inner` returns false at runtime.
WithFailMessage {
inner: Box<Self>,
message: String,
},
Invoke {
value: bool,
},
LocalIntEquals {
value: i64,
},
LocalIntLessOrEqual {
value: i64,
},
LookaheadTextEquals {
offset: isize,
text: String,
},
TextEquals(String),
TokenStartColumnEquals(usize),
ColumnLessThan(usize),
ColumnGreaterOrEqual(usize),
LookaheadNotEquals {
offset: isize,
token_name: String,
},
TokenPairAdjacent,
ContextChildRuleTextNotEquals {
rule_name: String,
text: String,
},
}
fn can_generate_parser_predicate(predicate: &PredicateTemplate) -> bool {
// A `<fail=...>` wrapper is transparent: generatability follows the inner.
matches!(
predicate_effective_template(predicate),
PredicateTemplate::Hook
| PredicateTemplate::True
| PredicateTemplate::False
| PredicateTemplate::FalseWithMessage { .. }
| PredicateTemplate::Invoke { .. }
| PredicateTemplate::LocalIntEquals { .. }
| PredicateTemplate::LocalIntLessOrEqual { .. }
| PredicateTemplate::LookaheadTextEquals { .. }
| PredicateTemplate::LookaheadNotEquals { .. }
| PredicateTemplate::TokenPairAdjacent
| PredicateTemplate::ContextChildRuleTextNotEquals { .. }
)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RuleArgTemplate {
Literal(i64),
InheritLocal,
}
/// Translates the lexer recognizer surface used by rendered test bodies onto
/// the `BaseLexer` hooks: `self.text()` / column accessors become position
/// -aware `_base` calls, `self.output()` becomes stdout.
fn translate_embedded_lexer_body(body: &str, position_expr: &str) -> io::Result<String> {
let mut out = replace_all(body.trim(), "self.output()", "std::io::stdout()");
out = replace_all(
&out,
"self.text()",
&format!("_base.token_text_until({position_expr})"),
);
out = replace_all(
&out,
"self.char_position_in_line()",
&format!("_base.column_at({position_expr})"),
);
out = replace_all(
&out,
"self.token_start_char_position_in_line()",
"_base.token_start_column()",
);
if out.contains("self.") {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported recognizer surface in embedded lexer body: {body}"),
));
}
Ok(out)
}
fn structural_embedded_lexer_actions(
data: &CodegenData<'_>,
) -> io::Result<Vec<((i32, i32), String)>> {
let mut actions = structural_actions(data)?
.into_iter()
.map(|action| {
Ok((
(
i32::try_from(action.rule_index).expect("rule index exceeds i32"),
i32::try_from(action.action_index).expect("action index exceeds i32"),
),
translate_embedded_lexer_body(&action.body, "action.position()")?,
))
})
.collect::<io::Result<Vec<_>>>()?;
actions.sort_by_key(|(coordinate, _)| *coordinate);
actions.dedup_by_key(|(coordinate, _)| *coordinate);
Ok(actions)
}
fn structural_embedded_lexer_predicates(
data: &CodegenData<'_>,
) -> io::Result<Vec<((usize, usize), String)>> {
structural_predicates(data)?
.into_iter()
.map(|predicate| {
Ok((
(predicate.rule_index, predicate.predicate_index),
translate_embedded_lexer_body(&predicate.body, "predicate.position()")?,
))
})
.collect()
}
fn render_embedded_lexer_action_method(actions: &[((i32, i32), String)]) -> String {
if actions
.iter()
.all(|(_, statement)| statement.trim().is_empty())
{
return String::new();
}
let mut arms = String::new();
for ((rule_index, action_index), statement) in actions {
writeln!(
arms,
" ({rule_index}, {action_index}) => {{ {statement} true }}"
)
.expect("writing to a string cannot fail");
}
arms.push_str(" _ => false,\n");
format!(
" fn run_action(_base: &mut BaseLexer<I>, action: antlr4_runtime::LexerCustomAction) -> bool {{\n #[allow(unused_imports)]\n use std::io::Write as _;\n match (action.rule_index(), action.action_index()) {{\n{arms} }}\n }}\n"
)
}
fn render_embedded_lexer_predicate_method(predicates: &[((usize, usize), String)]) -> String {
if predicates.is_empty() {
return String::new();
}
let mut arms = String::new();
for ((rule_index, pred_index), expression) in predicates {
writeln!(
arms,
" ({rule_index}, {pred_index}) => Some({{ {expression} }}),"
)
.expect("writing to a string cannot fail");
}
arms.push_str(" _ => None,\n");
format!(
" fn run_predicate(_base: &BaseLexer<I>, predicate: antlr4_runtime::LexerPredicate) -> Option<bool> {{\n match (predicate.rule_index(), predicate.pred_index()) {{\n{arms} }}\n }}\n"
)
}
fn reject_unsupported_lexer_action_templates(
actions: &[ActionTemplate],
allow_unsupported_only: bool,
) -> io::Result<()> {
if let Some(ActionTemplate::UnsupportedLexerAction { rule_name, body }) = actions
.iter()
.find(|action| matches!(action, ActionTemplate::UnsupportedLexerAction { .. }))
{
let has_supported_dispatch = actions.iter().any(lexer_action_template_needs_dispatch);
if allow_unsupported_only && !has_supported_dispatch {
return Ok(());
}
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"unsupported embedded lexer action in rule {rule_name}: {{{body}}}; \
rewrite target-specific actions as portable lexer commands where possible"
),
));
}
Ok(())
}
fn parse_lexer_action_block_template(body: &str) -> Option<ActionTemplate> {
parse_lexer_pop_mode_action(body)
}
fn parse_lexer_pop_mode_action(body: &str) -> Option<ActionTemplate> {
let body = body
.chars()
.filter(|ch| !ch.is_ascii_whitespace() && *ch != ';')
.collect::<String>();
matches!(
body.as_str(),
"popMode()"
| "this.popMode()"
| "if(!_modeStack.isEmpty()){popMode()}"
| "if(!this._modeStack.isEmpty()){popMode()}"
| "if(!_modeStack.isEmpty())popMode()"
| "if(!this._modeStack.isEmpty())popMode()"
)
.then_some(ActionTemplate::LexerPopMode)
}
fn one_line_action_body(body: &str) -> String {
const ACTION_SUMMARY_LIMIT: usize = 96;
let mut out = String::new();
for (index, part) in body.split_whitespace().enumerate() {
if index > 0 {
out.push(' ');
}
out.push_str(part);
if out.len() > ACTION_SUMMARY_LIMIT {
let mut limit = ACTION_SUMMARY_LIMIT;
while !out.is_char_boundary(limit) {
limit -= 1;
}
out.truncate(limit);
out.push_str("...");
break;
}
}
out
}
fn rust_block_comment_text(value: &str) -> String {
let mut out = String::new();
let mut cursor = 0;
while let Some(relative_index) = value[cursor..].find("*/") {
let index = cursor + relative_index;
out.push_str(&value[cursor..index]);
out.push_str("* /");
cursor = index + 2;
}
if cursor == 0 {
value.to_owned()
} else {
out.push_str(&value[cursor..]);
out
}
}
/// Attaches ANTLR's `<fail=...>` option to a predicate template so the runtime
/// can surface the grammar-supplied message when the predicate fails at runtime.
///
/// A constant-false predicate folds into `FalseWithMessage` (its own dedicated
/// variant). Any *other* template — a hook, lookahead, member, or local-int
/// predicate that can also return false at runtime — is wrapped in
/// `WithFailMessage` so the message is preserved rather than discarded; the
/// wrapper is transparent to evaluation (see `predicate_effective_template`) and
/// only contributes the failure message.
fn predicate_template_with_fail_message(
template: PredicateTemplate,
message: String,
) -> PredicateTemplate {
match template {
PredicateTemplate::False => PredicateTemplate::FalseWithMessage { message },
// Already message-carrying: replace the message (a later `<fail=...>`
// wins) rather than nesting wrappers.
PredicateTemplate::FalseWithMessage { .. } => {
PredicateTemplate::FalseWithMessage { message }
}
PredicateTemplate::WithFailMessage { inner, .. } => {
PredicateTemplate::WithFailMessage { inner, message }
}
other => PredicateTemplate::WithFailMessage {
inner: Box::new(other),
message,
},
}
}
/// The evaluation-relevant template, unwrapping a `WithFailMessage` wrapper.
/// The wrapper only carries a failure message; its runtime truth value and
/// codegen come from the inner template.
fn predicate_effective_template(template: &PredicateTemplate) -> &PredicateTemplate {
match template {
PredicateTemplate::WithFailMessage { inner, .. } => inner,
other => other,
}
}
/// The grammar-supplied `<fail=...>` message a template carries, if any.
fn predicate_template_fail_message(template: &PredicateTemplate) -> Option<&str> {
match template {
PredicateTemplate::FalseWithMessage { message }
| PredicateTemplate::WithFailMessage { message, .. } => Some(message),
_ => None,
}
}
/// Reports whether a predicate body is an untranslated ANTLR `<...>`
/// `StringTemplate` (a single template wrapper or a sequence of them), as
/// opposed to a native target-language predicate that merely contains a `<`
/// operator.
fn is_unsupported_string_template_body(body: &str) -> bool {
single_template_body(body).is_some() || template_sequence_bodies(body).is_some()
}
fn uses_alt_number_contexts(data: &CodegenData<'_>) -> bool {
let Some(semantic) = data.semantic else {
return false;
};
semantic
.unit
.options
.iter()
.any(|option| option.name.value == "contextSuperClass")
}
fn uses_structural_context_alt_numbers(data: &CodegenData<'_>) -> io::Result<bool> {
if data.semantic.is_none() {
return Ok(false);
}
let model = structural_embedded_model(data, false)?;
Ok(model.rules.iter().any(|rule| {
let left_recursive = rule
.alts
.iter()
.any(|alternative| alternative.is_lr_operator(&rule.name));
if !left_recursive {
return rule
.alts
.iter()
.map(|alternative| alternative.label.as_deref())
.collect::<BTreeSet<_>>()
.len()
> 1;
}
[false, true].into_iter().any(|operator| {
rule.alts
.iter()
.filter(|alternative| alternative.is_lr_operator(&rule.name) == operator)
.map(|alternative| alternative.label.as_deref())
.collect::<BTreeSet<_>>()
.len()
> 1
})
}))
}
fn parse_predicate_template(body: &str) -> Option<PredicateTemplate> {
let body = body.trim();
if let Some(inner) = single_template_body(body) {
return parse_predicate_template(inner);
}
match body {
"True()" => Some(PredicateTemplate::True),
"False()" => Some(PredicateTemplate::False),
r#"ParserPropertyCall({$parser}, "Property()")"# => Some(PredicateTemplate::True),
_ => parse_raw_boolean_predicate(body)
.or_else(|| parse_text_equals_predicate(body))
.or_else(|| parse_token_start_column_equals_predicate(body))
.or_else(|| parse_column_compare_predicate(body))
.or_else(|| parse_invoke_predicate(body))
.or_else(|| parse_val_equals_predicate(body))
.or_else(|| parse_raw_local_int_less_or_equal_predicate(body))
.or_else(|| parse_csharp_parser_predicate(body))
.or_else(|| parse_lt_equals_predicate(body))
.or_else(|| parse_la_not_equals_predicate(body)),
}
}
fn parse_predicate_template_with_patterns(
body: &str,
patterns: &SemPatternFile,
) -> io::Result<Option<PredicateTemplate>> {
parse_predicate_template_with_patterns_kind(body, patterns, SemanticsKind::ParserPredicate)
}
fn parse_predicate_template_with_patterns_kind(
body: &str,
patterns: &SemPatternFile,
kind: SemanticsKind,
) -> io::Result<Option<PredicateTemplate>> {
Ok(match parse_predicate_template(body) {
Some(template) => Some(template),
None => patterns.predicate_template(kind, body)?,
})
}
fn parse_raw_boolean_predicate(body: &str) -> Option<PredicateTemplate> {
match body {
"true" => return Some(PredicateTemplate::True),
"false" => return Some(PredicateTemplate::False),
_ => {}
}
let (equals, left, right) = if let Some((left, right)) = body.split_once("==") {
(true, left, right)
} else {
let (left, right) = body.split_once("!=")?;
(false, left, right)
};
let left = left.trim().parse::<i64>().ok()?;
let right = right.trim().parse::<i64>().ok()?;
let value = if equals { left == right } else { left != right };
Some(if value {
PredicateTemplate::True
} else {
PredicateTemplate::False
})
}
/// Returns the call body for an action made of exactly one target template.
fn single_template_body(body: &str) -> Option<&str> {
let body = body.trim();
if body.as_bytes().first() != Some(&b'<') {
return None;
}
let close = matching_template_close(body, 1)?;
(close + 1 == body.len()).then_some(&body[1..close])
}
/// Parses simple local integer argument predicates such as
/// `ValEquals("$i","2")`.
fn parse_val_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let arguments = body
.strip_prefix("ValEquals(")
.and_then(|value| value.strip_suffix(')'))
.map(split_template_arguments)?;
let [local, value] = arguments.as_slice() else {
return None;
};
if parse_template_string(local)? != "$i" {
return None;
}
Some(PredicateTemplate::LocalIntEquals {
value: parse_template_string(value)?.parse::<i64>().ok()?,
})
}
/// Parses raw ANTLR semantic predicates such as `5 >= $_p`.
///
/// The Java generator lowers these against the generated context field
/// `_localctx._p`. The metadata runtime does not execute target code, so the
/// generator records the literal bound and the rule-call argument table makes
/// the current `_p` value available while interpreting the predicate
/// transition.
fn parse_raw_local_int_less_or_equal_predicate(body: &str) -> Option<PredicateTemplate> {
let (value, local) = body.split_once(">=")?;
if local.trim() != "$_p" {
return None;
}
Some(PredicateTemplate::LocalIntLessOrEqual {
value: value.trim().parse::<i64>().ok()?,
})
}
/// Parses the runtime-testsuite helper that prints when a predicate is
/// evaluated before returning the wrapped boolean value.
fn parse_invoke_predicate(body: &str) -> Option<PredicateTemplate> {
let value = body.strip_suffix(":Invoke_pred()")?;
match value {
"True()" => Some(PredicateTemplate::Invoke { value: true }),
"False()" => Some(PredicateTemplate::Invoke { value: false }),
r#"ValEquals("$i","99")"# => Some(PredicateTemplate::Invoke { value: true }),
_ => None,
}
}
fn parse_csharp_parser_predicate(body: &str) -> Option<PredicateTemplate> {
match body.trim() {
"this.IsRightArrow()" | "this.IsRightShift()" | "this.IsRightShiftAssignment()" => {
Some(PredicateTemplate::TokenPairAdjacent)
}
"this.IsLocalVariableDeclaration()" => {
Some(PredicateTemplate::ContextChildRuleTextNotEquals {
rule_name: "local_variable_type".to_owned(),
text: "var".to_owned(),
})
}
_ => None,
}
}
fn parse_text_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let argument = body
.strip_prefix("TextEquals(")
.and_then(|value| value.strip_suffix(')'))?;
Some(PredicateTemplate::TextEquals(parse_template_string(
argument,
)?))
}
fn parse_token_start_column_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let argument = body
.strip_prefix("TokenStartColumnEquals(")
.and_then(|value| value.strip_suffix(')'))?;
Some(PredicateTemplate::TokenStartColumnEquals(
parse_template_string(argument)?.parse().ok()?,
))
}
/// Parses lexer column predicates serialized by upstream templates as
/// `<Column()> \< 2` or `<Column()> >= 2`.
fn parse_column_compare_predicate(body: &str) -> Option<PredicateTemplate> {
let rest = body
.trim()
.strip_prefix("<Column()>")
.or_else(|| body.trim().strip_prefix("Column()"))?
.trim_start();
let rest = rest.strip_prefix('\\').unwrap_or(rest).trim_start();
if let Some(value) = rest.strip_prefix('<') {
return Some(PredicateTemplate::ColumnLessThan(
value.trim().parse().ok()?,
));
}
Some(PredicateTemplate::ColumnGreaterOrEqual(
rest.strip_prefix(">=")?.trim().parse().ok()?,
))
}
fn parse_la_not_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let arguments = body
.strip_prefix("LANotEquals(")
.and_then(|value| value.strip_suffix(')'))
.map(split_template_arguments)?;
let [offset, token] = arguments.as_slice() else {
return None;
};
let offset = parse_template_string(offset)?.parse::<isize>().ok()?;
let token_name = parse_parser_token_argument(token)?;
Some(PredicateTemplate::LookaheadNotEquals { offset, token_name })
}
/// Parses `LTEquals` predicates that compare lookahead token text.
///
/// The runtime-testsuite passes the expected text as a quoted target-language
/// string literal, so the decoded `StringTemplate` argument may still contain
/// one nested quote pair.
fn parse_lt_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let arguments = body
.strip_prefix("LTEquals(")
.and_then(|value| value.strip_suffix(')'))
.map(split_template_arguments)?;
let [offset, text] = arguments.as_slice() else {
return None;
};
let offset = parse_template_string(offset)?.parse::<isize>().ok()?;
let text = parse_template_string(text)?;
let text = text
.strip_prefix('"')
.and_then(|value| value.strip_suffix('"'))
.unwrap_or(&text)
.to_owned();
Some(PredicateTemplate::LookaheadTextEquals { offset, text })
}
fn parse_parser_token_argument(argument: &str) -> Option<String> {
let body = argument
.trim()
.strip_prefix("{T<ParserToken(")?
.strip_suffix(")>}")?;
let parts = split_template_arguments(body);
let [_, token_name] = parts.as_slice() else {
return None;
};
parse_template_string(token_name)
}
/// Reads the lexer ATN to locate serialized custom action coordinates.
fn lexer_custom_actions(data: &CodegenData<'_>) -> io::Result<Vec<(i32, i32)>> {
let atn = data.lexer_atn()?;
Ok(atn
.lexer_actions()
.iter()
.filter_map(|action| match action {
LexerAction::Custom {
rule_index,
action_index,
} => Some((*rule_index, *action_index)),
_ => None,
})
.collect())
}
/// Reads the lexer ATN to locate semantic predicate coordinates.
fn lexer_predicate_transitions(data: &CodegenData<'_>) -> io::Result<Vec<(usize, usize)>> {
let atn = data.lexer_atn()?;
let mut predicates = Vec::new();
for state in atn.states() {
for transition in &state.transitions {
if let LexerTransition::Predicate {
rule_index,
pred_index,
..
} = transition
{
predicates.push((*rule_index, *pred_index));
}
}
}
Ok(predicates)
}
/// Reads the packed parser ATN to locate semantic predicate coordinates.
fn parser_predicate_transitions(data: &CodegenData<'_>) -> io::Result<Vec<(usize, usize)>> {
let atn = data.parser_atn()?;
let mut predicates = Vec::new();
for state in atn.states() {
for transition in state.transitions() {
if let ParserTransitionData::Predicate {
rule_index,
pred_index,
..
} = transition.data()
{
predicates.push((rule_index, pred_index));
}
}
}
Ok(predicates)
}
/// Reads the parser ATN to locate action-transition source states.
fn parser_action_states(data: &CodegenData<'_>) -> io::Result<Vec<usize>> {
let atn = data.parser_atn()?;
let mut states = Vec::new();
for state in atn.states() {
if state
.transitions()
.iter()
.any(|transition| matches!(transition.data(), ParserTransitionData::Action { .. }))
{
states.push(state.state_number());
}
}
Ok(states)
}
/// Reads the parser ATN action transitions keyed by source state.
fn parser_action_state_rules(data: &CodegenData<'_>) -> io::Result<BTreeMap<usize, usize>> {
let atn = data.parser_atn()?;
let mut states = BTreeMap::new();
for state in atn.states() {
for transition in state.transitions() {
if let ParserTransitionData::Action { rule_index, .. } = transition.data() {
states.insert(state.state_number(), rule_index);
}
}
}
Ok(states)
}
/// The parser ATN action states that ANTLR *synthesized* (during left-recursion
/// elimination and similar rewrites) rather than the author writing them.
///
/// Provenance marks transformed action elements directly. Any action transition
/// not represented by a structural element is synthetic as well.
fn synthetic_parser_action_states(data: &CodegenData<'_>) -> io::Result<BTreeSet<usize>> {
let actions = structural_actions(data)?;
let represented = actions
.iter()
.map(|action| action.state)
.collect::<BTreeSet<_>>();
let mut synthetic = actions
.into_iter()
.filter_map(|action| (!action.authored).then_some(action.state))
.collect::<BTreeSet<_>>();
synthetic.extend(
parser_action_states(data)?
.into_iter()
.filter(|state| !represented.contains(state)),
);
Ok(synthetic)
}
fn empty_parser_action_states(data: &CodegenData<'_>) -> io::Result<BTreeSet<usize>> {
Ok(structural_actions(data)?
.into_iter()
.filter_map(|action| {
(action.authored && action.body.trim().is_empty()).then_some(action.state)
})
.collect())
}
fn structural_parser_rule_args(
data: &CodegenData<'_>,
) -> io::Result<Vec<(usize, usize, RuleArgTemplate)>> {
Ok(structural_rule_calls(data)?
.into_iter()
.filter_map(|call| {
let template = parse_rule_arg_template(call.arguments.as_deref()?)?;
Some((call.state, call.target_rule_index, template))
})
.collect())
}
fn parse_rule_arg_template(value: &str) -> Option<RuleArgTemplate> {
let value = value.trim();
value.parse::<i64>().map_or_else(
|_| {
if matches!(value, "true" | "false") {
Some(RuleArgTemplate::Literal(i64::from(value == "true")))
} else if value == r#"<VarRef("i")>"# {
Some(RuleArgTemplate::InheritLocal)
} else {
None
}
},
|value| Some(RuleArgTemplate::Literal(value)),
)
}
/// Emits the generated lexer action dispatcher for structurally bound custom
/// actions in the compiled lexer.
fn render_lexer_action_method(actions: &[((i32, i32), ActionTemplate)]) -> String {
if actions.is_empty() {
return String::new();
}
let mut comments = String::new();
for (_, template) in actions {
if let ActionTemplate::UnsupportedLexerAction { rule_name, body } = template {
writeln!(
comments,
" {}",
render_unsupported_lexer_action_comment(rule_name, body)
)
.expect("writing to a string cannot fail");
}
}
if !lexer_actions_need_dispatch(actions) {
return comments;
}
let mut arms = String::new();
for ((rule_index, action_index), template) in actions {
if !lexer_action_template_needs_dispatch(template) {
continue;
}
let statement = render_lexer_action_statement(template);
writeln!(
arms,
" ({rule_index}, {action_index}) => {{ {statement} true }}"
)
.expect("writing to a string cannot fail");
}
arms.push_str(" _ => false,\n");
format!(
"{comments} fn run_action(_base: &mut BaseLexer<I>, action: antlr4_runtime::LexerCustomAction) -> bool {{\n match (action.rule_index(), action.action_index()) {{\n{arms} }}\n }}\n"
)
}
fn lexer_actions_need_dispatch(actions: &[((i32, i32), ActionTemplate)]) -> bool {
actions
.iter()
.any(|(_, template)| lexer_action_template_needs_dispatch(template))
}
fn lexer_action_template_needs_dispatch(template: &ActionTemplate) -> bool {
match template {
ActionTemplate::Hook(_) | ActionTemplate::UnsupportedLexerAction { .. } => false,
ActionTemplate::LexerPopMode => !render_lexer_action_statement(template).is_empty(),
}
}
/// Renders one supported lexer target-template action as Rust code.
fn render_lexer_action_statement(template: &ActionTemplate) -> String {
match template {
ActionTemplate::LexerPopMode => "_base.pop_mode();".to_owned(),
ActionTemplate::Hook(_) => String::new(),
ActionTemplate::UnsupportedLexerAction { rule_name, body } => {
render_unsupported_lexer_action_comment(rule_name, body)
}
}
}
fn render_unsupported_lexer_action_comment(rule_name: &str, body: &str) -> String {
format!(
"/* TODO unsupported embedded lexer action in rule {}: {{{}}}; rewrite target-specific actions as portable lexer commands where possible */",
rust_block_comment_text(rule_name),
rust_block_comment_text(body)
)
}
/// Emits the generated lexer predicate dispatcher for structurally bound
/// predicate coordinates in the compiled lexer.
fn render_lexer_predicate_method(
predicates: &[((usize, usize), PredicateTemplate)],
sem_unknown: SemUnknownPolicy,
) -> String {
if predicates.is_empty() {
return String::new();
}
let mut arms = String::new();
for ((rule_index, pred_index), template) in predicates {
let statement = if matches!(template, PredicateTemplate::Hook) {
"None".to_owned()
} else {
format!("Some({})", render_lexer_predicate_expression(template))
};
writeln!(
arms,
" ({rule_index}, {pred_index}) => {{ {statement} }}"
)
.expect("writing to a string cannot fail");
}
// The catch-all arm is the unknown-lexer-predicate handler for any
// coordinate this grammar left untranslated. `--sem-unknown=assume-false`
// must flip it to `false` here too; otherwise a mixed lexer (one translated
// predicate plus an uncovered coordinate) keeps the uncovered guard viable
// even though the manifest promised assume-false.
let default_arm = if sem_unknown == SemUnknownPolicy::AssumeFalse {
" _ => Some(false),\n"
} else if sem_unknown == SemUnknownPolicy::Hook {
" _ => None,\n"
} else {
" _ => Some(true),\n"
};
arms.push_str(default_arm);
format!(
" fn run_predicate(_base: &BaseLexer<I>, predicate: antlr4_runtime::LexerPredicate) -> Option<bool> {{\n match (predicate.rule_index(), predicate.pred_index()) {{\n{arms} }}\n }}\n"
)
}
fn render_lexer_predicate_expression(template: &PredicateTemplate) -> String {
match template {
// A `<fail=...>` wrapper is transparent to evaluation; render its inner.
PredicateTemplate::WithFailMessage { inner, .. } => {
render_lexer_predicate_expression(inner)
}
PredicateTemplate::True => "true".to_owned(),
PredicateTemplate::False => "false".to_owned(),
PredicateTemplate::TextEquals(value) => format!(
"_base.token_text_until(predicate.position()) == \"{}\"",
rust_string(value)
),
PredicateTemplate::TokenStartColumnEquals(value) => {
format!("_base.token_start_column() == {value}")
}
PredicateTemplate::ColumnLessThan(value) => {
format!("_base.column_at(predicate.position()) < {value}")
}
PredicateTemplate::ColumnGreaterOrEqual(value) => {
format!("_base.column_at(predicate.position()) >= {value}")
}
PredicateTemplate::Hook
| PredicateTemplate::Invoke { .. }
| PredicateTemplate::FalseWithMessage { .. }
| PredicateTemplate::LocalIntEquals { .. }
| PredicateTemplate::LocalIntLessOrEqual { .. }
| PredicateTemplate::LookaheadTextEquals { .. }
| PredicateTemplate::LookaheadNotEquals { .. }
| PredicateTemplate::TokenPairAdjacent
| PredicateTemplate::ContextChildRuleTextNotEquals { .. } => {
unreachable!("lookahead parser predicates are not lexer predicates")
}
}
}
/// Reports whether a parser action source state carries an `assume-true` /
/// `assume-false` override. Such a coordinate is a documented silent no-op:
/// it must NOT fall through to the `parser_action_hook` catch-all (which fails
/// loud under the Error policy or runs a user side effect). It gets an explicit
/// empty arm instead. A `hook` (or `error`) override is excluded here so it
/// still routes to the hook.
fn parser_action_assume_overridden(
patterns: &SemPatternFile,
data: &CodegenData<'_>,
action_state_rules: &BTreeMap<usize, usize>,
state: usize,
) -> bool {
let rule_name = action_state_rules
.get(&state)
.and_then(|rule| data.rule_names.get(*rule).map(String::as_str));
patterns
.coordinate_override(SemanticsKind::ParserAction, rule_name, None, Some(state))
.is_some_and(|override_| {
matches!(
override_.dispose,
CoordinateDispose::AssumeTrue | CoordinateDispose::AssumeFalse
)
})
}
fn render_parser_action_method(has_action_states: bool, noop_states: &BTreeSet<usize>) -> String {
if !has_action_states {
return " fn run_action(&mut self, _action: antlr4_runtime::ParserAction, _tree: antlr4_runtime::ParseTree) {}\n"
.to_owned();
}
let mut arms = String::new();
for state in noop_states {
writeln!(arms, " {state} => {{}}").expect("writing to a string cannot fail");
}
arms.push_str(" _ => { let _ = self.base.parser_action_hook(action, tree); }\n");
format!(
" fn run_action(&mut self, action: antlr4_runtime::ParserAction, tree: antlr4_runtime::ParseTree) {{\n match action.source_state() {{\n{arms} }}\n }}\n"
)
}
fn likely_parser_entry_rule_indices(data: &CodegenData<'_>) -> io::Result<Vec<usize>> {
if let Some(semantic) = data.semantic {
return Ok(semantic
.entry_rules
.iter()
.map(|rule| semantic.recognizer.rule_numbers[rule])
.collect());
}
let atn = data.parser_atn()?;
Ok(likely_parser_entry_rule_indices_from_atn(
atn,
data.rule_names.len(),
))
}
fn likely_parser_entry_rule_indices_from_atn(atn: &ParserAtn, rule_count: usize) -> Vec<usize> {
let mut called_by_other_rule = vec![false; rule_count];
for state in atn.states() {
for transition in state.transitions() {
let ParserTransitionData::Rule { rule_index, .. } = transition.data() else {
continue;
};
if rule_index >= rule_count || state.rule_index() == Some(rule_index) {
continue;
}
called_by_other_rule[rule_index] = true;
}
}
called_by_other_rule
.iter()
.enumerate()
.filter_map(|(index, called)| (!called).then_some(index))
.collect()
}
/// Renders the generated parser type rustdoc that surfaces callable rule methods.
fn render_parser_rustdoc(
public_rule_method_names: &[String],
entry_rule_indices: &[usize],
) -> String {
let all_method_capacity = public_rule_method_names
.iter()
.map(|method| method.len() + "/// - `()`\n".len())
.sum::<usize>();
let entry_method_capacity = entry_rule_indices
.iter()
.filter_map(|index| public_rule_method_names.get(*index))
.map(|method| method.len() + "/// - `()`\n".len())
.sum::<usize>();
let mut out = String::with_capacity(384 + all_method_capacity + entry_method_capacity);
writeln!(
out,
"/// Generated parser. Each grammar rule is exposed as a public method."
)
.expect("writing to a string cannot fail");
writeln!(out, "///").expect("writing to a string cannot fail");
writeln!(
out,
"/// Pick an entry-rule method that matches the grammar's intended"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"/// top-level construct for the input being parsed. The generator can"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"/// identify rules that are not called by another rule, but it cannot"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"/// infer the semantic choice between multiple candidates."
)
.expect("writing to a string cannot fail");
if !entry_rule_indices.is_empty() {
writeln!(out, "///").expect("writing to a string cannot fail");
writeln!(
out,
"/// Likely parser entry-rule methods (not called by other rules):"
)
.expect("writing to a string cannot fail");
for index in entry_rule_indices {
let Some(method_name) = public_rule_method_names.get(*index) else {
continue;
};
writeln!(out, "/// - `{method_name}()`").expect("writing to a string cannot fail");
}
}
if !public_rule_method_names.is_empty() {
writeln!(out, "///").expect("writing to a string cannot fail");
writeln!(out, "/// All parser rule methods:").expect("writing to a string cannot fail");
for method_name in public_rule_method_names {
writeln!(out, "/// - `{method_name}()`").expect("writing to a string cannot fail");
}
}
out
}
/// Renders static grammar metadata shared by generated lexers and parsers.
fn render_metadata(grammar_name: &str, data: &CodegenData<'_>) -> String {
render_metadata_with_atn(grammar_name, data, &data.lexer_atn_words)
}
/// Parser modules carry the versioned packed ATN separately, so retaining the
/// legacy serialized integer stream in metadata would duplicate the artifact.
fn render_parser_metadata(grammar_name: &str, data: &CodegenData<'_>) -> String {
render_metadata_with_atn(grammar_name, data, &[])
}
fn render_metadata_with_atn(
grammar_name: &str,
data: &CodegenData<'_>,
serialized_atn: &[i32],
) -> String {
format!(
"pub static METADATA: GrammarMetadata = GrammarMetadata::new(\n \"{}\",\n &{},\n &{},\n &{},\n &{},\n &{},\n &{},\n &{},\n);\n\npub fn metadata() -> &'static GrammarMetadata {{\n &METADATA\n}}\n\npub fn rule_names() -> &'static [&'static str] {{\n METADATA.rule_names()\n}}\n",
rust_string(grammar_name),
render_str_slice(&data.rule_names),
render_option_str_slice(&data.literal_names),
render_option_str_slice(&data.symbolic_names),
render_empty_option_str_slice(max_len(&data.literal_names, &data.symbolic_names)),
render_str_slice(&data.channel_names),
render_str_slice(&data.mode_names),
render_i32_slice(serialized_atn)
)
}
/// Renders token constants from symbolic token names while avoiding duplicate
/// Rust identifiers after sanitization.
fn render_token_constants(data: &CodegenData<'_>) -> String {
let mut out = String::from("pub const EOF: i32 = antlr4_runtime::TOKEN_EOF;\n");
let mut seen = BTreeSet::new();
for (index, name) in data.symbolic_names.iter().enumerate() {
let Some(name) = name else { continue };
let ident = rust_const_name(name);
if ident == "EOF" || !seen.insert(ident.clone()) {
continue;
}
writeln!(out, "pub const {ident}: i32 = {index};")
.expect("writing to a string cannot fail");
}
out
}
/// Renders rule-index constants from grammar rule names.
fn render_rule_constants(data: &CodegenData<'_>) -> String {
let mut out = String::new();
for (index, name) in data.rule_names.iter().enumerate() {
writeln!(
out,
"pub const RULE_{}: usize = {index};",
rust_const_name(name)
)
.expect("writing to a string cannot fail");
}
out
}
/// Renders an `&[Option<&str>]` expression for literal or symbolic names.
fn render_option_str_slice(values: &[Option<String>]) -> String {
let items = values
.iter()
.map(|value| {
value.as_ref().map_or_else(
|| "None".to_owned(),
|value| format!("Some(\"{}\")", rust_string(value)),
)
})
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders an empty optional string table with a fixed length.
fn render_empty_option_str_slice(len: usize) -> String {
let items = (0..len).map(|_| "None").collect::<Vec<_>>().join(", ");
format!("[{items}]")
}
/// Renders an `&[&str]` expression for rule/channel/mode names.
fn render_str_slice(values: &[String]) -> String {
let items = values
.iter()
.map(|value| format!("\"{}\"", rust_string(value)))
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders a line-wrapped `&[i32]` expression for serialized ATN data.
fn render_i32_slice(values: &[i32]) -> String {
let items = values
.iter()
.map(i32::to_string)
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders a versioned packed parser ATN word stream.
fn render_u32_slice(values: &[u32]) -> String {
let items = values
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders an inline `[(i32, i32); N]` expression for generated token-set
/// matches.
fn render_i32_ranges(values: &[(i32, i32)]) -> String {
let items = values
.iter()
.map(|(start, stop)| format!("({start}, {stop})"))
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
fn render_i32_match_patterns(values: &[(i32, i32)]) -> String {
values
.iter()
.map(|(start, stop)| {
if start == stop {
start.to_string()
} else {
format!("{start}..={stop}")
}
})
.collect::<Vec<_>>()
.join(" | ")
}
/// Renders parser predicate metadata shared by generated predicate checks.
#[allow(dead_code)]
fn render_parser_predicate_constant(
predicates: &[((usize, usize), PredicateTemplate)],
data: &CodegenData<'_>,
) -> io::Result<String> {
let predicates = render_parser_predicate_array(predicates, data)?;
Ok(format!(
"#[allow(dead_code)]\nconst PARSER_PREDICATES: &[(usize, usize, antlr4_runtime::ParserPredicate)] = &{predicates};\n"
))
}
fn render_parser_semantics_function(
predicates: &[((usize, usize), PredicateTemplate)],
data: &CodegenData<'_>,
) -> io::Result<String> {
let predicate_builders = render_parser_semir_predicate_builders(predicates, data)?;
Ok(format!(
r#"fn parser_semantics() -> &'static antlr4_runtime::ParserSemantics {{
static SEMANTICS_CELL: OnceLock<antlr4_runtime::ParserSemantics> = OnceLock::new();
SEMANTICS_CELL.get_or_init(|| {{
let mut ir = antlr4_runtime::semir::SemIr::new();
let mut predicates = Vec::new();
{predicate_builders}
let actions = Vec::new();
antlr4_runtime::ParserSemantics {{ ir, predicates, actions }}
}})
}}
"#
))
}
fn lexer_typed_hook_mappings(
data: &CodegenData<'_>,
patterns: &SemPatternFile,
actions: &[((i32, i32), ActionTemplate)],
) -> io::Result<Vec<LexerTypedHookMapping>> {
let mut mappings = actions
.iter()
.filter_map(|((rule_index, action_index), template)| {
let ActionTemplate::Hook(call) = template else {
return None;
};
Some(LexerTypedHookMapping {
rule_index: usize::try_from(*rule_index).ok()?,
coordinate_index: usize::try_from(*action_index).ok()?,
kind: LexerTypedHookKind::Action,
method_name: rust_function_name(&call.name),
call: call.clone(),
})
})
.collect::<Vec<_>>();
for predicate in structural_predicates(data)? {
if let Some(call) =
patterns.hook_helper_call(SemanticsKind::LexerPredicate, &predicate.body)?
{
mappings.push(LexerTypedHookMapping {
rule_index: predicate.rule_index,
coordinate_index: predicate.predicate_index,
kind: LexerTypedHookKind::Predicate,
method_name: rust_function_name(&call.name),
call,
});
}
}
let predicate_names = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Predicate)
.map(|mapping| mapping.method_name.clone())
.collect::<BTreeSet<_>>();
let action_names = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Action)
.map(|mapping| mapping.method_name.clone())
.collect::<BTreeSet<_>>();
for mapping in &mut mappings {
if predicate_names.contains(&mapping.method_name)
&& action_names.contains(&mapping.method_name)
{
mapping.method_name.push_str(match mapping.kind {
LexerTypedHookKind::Predicate => "_pred",
LexerTypedHookKind::Action => "_action",
});
}
}
validate_lexer_typed_hook_signatures(&mappings)?;
mappings.sort_by_key(|mapping| {
(
mapping.rule_index,
mapping.coordinate_index,
matches!(mapping.kind, LexerTypedHookKind::Action),
)
});
mappings.dedup();
Ok(mappings)
}
fn validate_lexer_typed_hook_signatures(mappings: &[LexerTypedHookMapping]) -> io::Result<()> {
let mut signatures = BTreeMap::<(&str, LexerTypedHookKind), Vec<SemanticLiteralKind>>::new();
for mapping in mappings {
let signature = mapping
.call
.arguments
.iter()
.map(semantic_literal_kind)
.collect::<Vec<_>>();
match signatures.entry((&mapping.method_name, mapping.kind)) {
Entry::Occupied(entry) if entry.get() != &signature => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"typed lexer semantic helper {} has conflicting literal signatures {:?} and {signature:?}",
mapping.call.name,
entry.get()
),
));
}
Entry::Occupied(_) => {}
Entry::Vacant(entry) => {
entry.insert(signature);
}
}
}
Ok(())
}
fn render_lexer_typed_hook_adapter(type_name: &str, mappings: &[LexerTypedHookMapping]) -> String {
if mappings.is_empty() {
return String::new();
}
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
let mut methods = BTreeMap::<(String, bool), Vec<SemanticLiteral>>::new();
for mapping in mappings {
methods
.entry((
mapping.method_name.clone(),
mapping.kind == LexerTypedHookKind::Predicate,
))
.or_insert_with(|| mapping.call.arguments.clone());
}
let method_decls = methods
.iter()
.map(|((method, predicate), arguments)| {
let arguments = render_semantic_method_arguments(arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let result = if *predicate { " -> bool" } else { "" };
format!(
" fn {method}<I>(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>{separator}{arguments}){result}\n where\n I: antlr4_runtime::CharStream;"
)
})
.collect::<Vec<_>>()
.join("\n\n");
let predicate_arms = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Predicate)
.map(|mapping| {
let rule = mapping.rule_index;
let index = mapping.coordinate_index;
let arguments = render_semantic_call_arguments(&mapping.call.arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let method = &mapping.method_name;
let call = format!("self.0.{method}(ctx{separator}{arguments})");
let call = if mapping.call.negated {
format!("!{call}")
} else {
call
};
format!(" ({rule}, {index}) => Some({call}),")
})
.collect::<Vec<_>>()
.join("\n");
let action_arms = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Action)
.map(|mapping| {
let rule = mapping.rule_index;
let index = mapping.coordinate_index;
let arguments = render_semantic_call_arguments(&mapping.call.arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let method = &mapping.method_name;
format!(
" ({rule}, {index}) => {{ self.0.{method}(ctx{separator}{arguments}); true }}"
)
})
.collect::<Vec<_>>()
.join("\n");
format!(
r#"pub trait {trait_name}: Sized {{
{method_decls}
fn token_emitted(&mut self, _token: antlr4_runtime::TokenView<'_>) {{}}
}}
#[derive(Clone, Debug, Default)]
pub struct {adapter_name}<T>(pub T);
impl<T> {adapter_name}<T> {{
pub const fn new(inner: T) -> Self {{ Self(inner) }}
}}
impl<T> antlr4_runtime::SemanticHooks for {adapter_name}<T>
where
T: {trait_name},
{{
fn lexer_sempred<I>(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>, rule_index: usize, pred_index: usize) -> Option<bool>
where
I: antlr4_runtime::CharStream,
{{
match (rule_index, pred_index) {{
{predicate_arms}
_ => None,
}}
}}
fn lexer_action<I>(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>, action: antlr4_runtime::LexerCustomAction) -> bool
where
I: antlr4_runtime::CharStream,
{{
let Ok(rule_index) = usize::try_from(action.rule_index()) else {{ return false; }};
let Ok(action_index) = usize::try_from(action.action_index()) else {{ return false; }};
match (rule_index, action_index) {{
{action_arms}
_ => false,
}}
}}
fn lexer_token_emitted(&mut self, token: antlr4_runtime::TokenView<'_>) {{
self.0.token_emitted(token);
}}
}}
"#
)
}
fn parser_typed_hook_mappings(
data: &CodegenData<'_>,
patterns: &SemPatternFile,
) -> io::Result<Vec<TypedHookMapping>> {
let mut mappings = Vec::new();
for predicate in structural_predicates(data)? {
push_typed_hook_mapping(
data,
patterns,
predicate.rule_index,
predicate.predicate_index,
&predicate.body,
&mut mappings,
)?;
}
mappings.sort_by_key(|mapping| (mapping.rule_index, mapping.pred_index));
mappings.dedup();
validate_typed_hook_signatures(&mappings)?;
Ok(mappings)
}
fn push_typed_hook_mapping(
data: &CodegenData<'_>,
patterns: &SemPatternFile,
rule_index: usize,
pred_index: usize,
body: &str,
mappings: &mut Vec<TypedHookMapping>,
) -> io::Result<()> {
let helper_call = parse_semantic_helper_call(body, SemanticsKind::ParserPredicate);
let forced_hook = patterns
.coordinate_predicate_template(
SemanticsKind::ParserPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
)
.is_some_and(|template| matches!(template, Some(PredicateTemplate::Hook)));
let parsed = parse_predicate_template_with_patterns(body, patterns)?;
if let Some(call) = helper_call
&& (forced_hook || parsed.is_none() || matches!(parsed, Some(PredicateTemplate::Hook)))
{
mappings.push(TypedHookMapping {
rule_index,
pred_index,
method_name: typed_hook_predicate_method_name(&call.name),
call,
});
}
Ok(())
}
/// Reserved name of the fixed action-hook method emitted on the typed-hook
/// trait. A predicate-helper method must not normalize to this, or the trait
/// would declare two `custom_action` methods (Rust has no arity overloading).
const TYPED_HOOK_ACTION_METHOD: &str = "custom_action";
/// The typed-hook trait method name for a bare predicate helper, disambiguated
/// so it never collides with the fixed [`TYPED_HOOK_ACTION_METHOD`]. A grammar
/// helper literally named `customAction()` / `custom_action()` normalizes to
/// `custom_action`; suffix it with `_pred` so the generated trait compiles.
fn typed_hook_predicate_method_name(helper: &str) -> String {
let name = rust_function_name(helper);
if name == TYPED_HOOK_ACTION_METHOD {
format!("{name}_pred")
} else {
name
}
}
const fn semantic_literal_kind(literal: &SemanticLiteral) -> SemanticLiteralKind {
match literal {
SemanticLiteral::String(_) => SemanticLiteralKind::String,
SemanticLiteral::Bool(_) => SemanticLiteralKind::Bool,
SemanticLiteral::Integer(_) => SemanticLiteralKind::Integer,
}
}
fn validate_typed_hook_signatures(mappings: &[TypedHookMapping]) -> io::Result<()> {
let mut signatures = BTreeMap::<&str, Vec<SemanticLiteralKind>>::new();
for mapping in mappings {
let signature = mapping
.call
.arguments
.iter()
.map(semantic_literal_kind)
.collect::<Vec<_>>();
match signatures.entry(&mapping.method_name) {
Entry::Occupied(entry) if entry.get() != &signature => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"typed semantic helper {} has conflicting literal signatures {:?} and {signature:?}",
mapping.call.name,
entry.get()
),
));
}
Entry::Occupied(_) => {}
Entry::Vacant(entry) => {
entry.insert(signature);
}
}
}
Ok(())
}
fn render_semantic_method_arguments(arguments: &[SemanticLiteral]) -> String {
arguments
.iter()
.enumerate()
.map(|(index, literal)| {
let ty = match literal {
SemanticLiteral::String(_) => "&str",
SemanticLiteral::Bool(_) => "bool",
SemanticLiteral::Integer(_) => "i64",
};
format!("arg{index}: {ty}")
})
.collect::<Vec<_>>()
.join(", ")
}
fn render_semantic_call_arguments(arguments: &[SemanticLiteral]) -> String {
arguments
.iter()
.map(|literal| match literal {
SemanticLiteral::String(value) => format!("\"{}\"", rust_string(value)),
SemanticLiteral::Bool(value) => value.to_string(),
SemanticLiteral::Integer(value) => value.to_string(),
})
.collect::<Vec<_>>()
.join(", ")
}
fn render_typed_hook_adapter(type_name: &str, mappings: &[TypedHookMapping]) -> String {
if mappings.is_empty() {
return String::new();
}
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
let mut methods = BTreeMap::new();
for mapping in mappings {
methods
.entry(mapping.method_name.clone())
.or_insert_with(|| mapping.call.arguments.clone());
}
let method_decls = methods
.iter()
.map(|(method, arguments)| {
let arguments = render_semantic_method_arguments(arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
format!(
" fn {method}<L>(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>{separator}{arguments}) -> bool\n where\n L: TokenSource;"
)
})
.collect::<Vec<_>>()
.join("\n\n");
let arms = mappings
.iter()
.map(|mapping| {
let rule_index = mapping.rule_index;
let pred_index = mapping.pred_index;
let method = &mapping.method_name;
let arguments = render_semantic_call_arguments(&mapping.call.arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let call = format!("self.0.{method}(ctx{separator}{arguments})");
let call = if mapping.call.negated {
format!("!{call}")
} else {
call
};
format!(" ({rule_index}, {pred_index}) => Some({call}),")
})
.collect::<Vec<_>>()
.join("\n");
format!(
r#"pub trait {trait_name}: Sized {{
{method_decls}
/// Handles a committed parser action routed to the typed hook. Return
/// `true` when the action is handled so it satisfies a `hook`/`error`
/// unknown-semantic policy; the default no-op returns `false` (unhandled),
/// which fails loud under those policies.
fn custom_action<L>(&mut self, _ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, _action: antlr4_runtime::ParserAction) -> bool
where
L: TokenSource,
{{
false
}}
}}
#[derive(Clone, Copy, Debug, Default)]
pub struct {adapter_name}<T>(pub T);
impl<T> {adapter_name}<T> {{
pub const fn new(inner: T) -> Self {{ Self(inner) }}
}}
impl<T> antlr4_runtime::SemanticHooks for {adapter_name}<T>
where
T: {trait_name},
{{
fn sempred<L>(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, rule_index: usize, pred_index: usize) -> Option<bool>
where
L: TokenSource,
{{
match (rule_index, pred_index) {{
{arms}
_ => None,
}}
}}
fn action<L>(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, action: antlr4_runtime::ParserAction) -> bool
where
L: TokenSource,
{{
self.0.custom_action(ctx, action)
}}
}}
"#
)
}
fn render_parser_semir_predicate_builders(
predicates: &[((usize, usize), PredicateTemplate)],
data: &CodegenData<'_>,
) -> io::Result<String> {
let mut out = String::new();
for ((rule_index, pred_index), predicate) in predicates {
let expr = render_parser_semir_predicate_expr(predicate, data)?;
// Carry the `<fail=...>` message for ANY predicate that supplies one, not
// only a constant-false one — a hook/lookahead/member predicate that
// returns false at runtime should surface the grammar's fail text too.
let failure_message = predicate_template_fail_message(predicate).map_or_else(
|| "None".to_owned(),
|message| format!("Some(\"{}\")", rust_string(message)),
);
writeln!(
out,
" let __expr = {expr};\n predicates.push(antlr4_runtime::ParserSemanticPredicate {{ rule_index: {rule_index}, pred_index: {pred_index}, expr: __expr, failure_message: {failure_message} }});"
)
.expect("writing to a string cannot fail");
}
Ok(out)
}
#[allow(clippy::too_many_lines)]
fn render_parser_semir_predicate_expr(
predicate: &PredicateTemplate,
data: &CodegenData<'_>,
) -> io::Result<String> {
match predicate {
// A `<fail=...>` wrapper is transparent to evaluation; lower its inner.
PredicateTemplate::WithFailMessage { inner, .. } => {
render_parser_semir_predicate_expr(inner, data)
}
PredicateTemplate::Hook => Ok(
"ir.expr(antlr4_runtime::semir::PExpr::Hook(antlr4_runtime::semir::HookId::new(0)))"
.to_owned(),
),
PredicateTemplate::True => {
Ok("ir.expr(antlr4_runtime::semir::PExpr::Bool(true))".to_owned())
}
PredicateTemplate::False | PredicateTemplate::FalseWithMessage { .. } => {
Ok("ir.expr(antlr4_runtime::semir::PExpr::Bool(false))".to_owned())
}
PredicateTemplate::Invoke { value } => Ok(format!(
"ir.expr(antlr4_runtime::semir::PExpr::EvalTrace({value}))"
)),
PredicateTemplate::LocalIntEquals { value } => Ok(render_local_arg_semir_cmp("Eq", *value)),
PredicateTemplate::LocalIntLessOrEqual { value } => {
Ok(render_local_arg_semir_cmp("Le", *value))
}
PredicateTemplate::LookaheadTextEquals { offset, text } => Ok(format!(
"{{ let __actual = ir.expr(antlr4_runtime::semir::PExpr::TokenText({offset})); let __text = ir.intern(\"{}\"); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Str(__text)); ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::Eq, __actual, __expected)) }}",
rust_string(text)
)),
PredicateTemplate::LookaheadNotEquals { offset, token_name } => {
let token_type = token_type_for_name(data, token_name).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate token {token_name}"),
)
})?;
Ok(format!(
"{{ let __actual = ir.expr(antlr4_runtime::semir::PExpr::La({offset})); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Int({token_type})); ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::Ne, __actual, __expected)) }}"
))
}
PredicateTemplate::TokenPairAdjacent => {
Ok("ir.expr(antlr4_runtime::semir::PExpr::TokenIndexAdjacent)".to_owned())
}
PredicateTemplate::ContextChildRuleTextNotEquals { rule_name, text } => {
let rule_index = data
.rule_names
.iter()
.position(|name| name == rule_name)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate rule {rule_name}"),
)
})?;
Ok(format!(
"{{ let __actual = ir.expr(antlr4_runtime::semir::PExpr::CtxRuleText({rule_index})); let __text = ir.intern(\"{}\"); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Str(__text)); ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::Ne, __actual, __expected)) }}",
rust_string(text)
))
}
PredicateTemplate::TextEquals(_)
| PredicateTemplate::TokenStartColumnEquals(_)
| PredicateTemplate::ColumnLessThan(_)
| PredicateTemplate::ColumnGreaterOrEqual(_) => Err(io::Error::new(
io::ErrorKind::InvalidData,
"lexer-only predicate cannot be lowered for parser SemIR",
)),
}
}
fn render_local_arg_semir_cmp(op: &str, value: i64) -> String {
format!(
"{{ let __local = ir.expr(antlr4_runtime::semir::PExpr::LocalArg); let __absent = ir.expr(antlr4_runtime::semir::PExpr::IsNull(__local)); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Int({value})); let __comparison = ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::{op}, __local, __expected)); ir.expr(antlr4_runtime::semir::PExpr::Or([__absent, __comparison].into())) }}"
)
}
/// Renders parser predicate metadata as an inline slice consumed by the runtime
/// parser interpreter.
#[allow(dead_code)]
fn render_parser_predicate_array(
predicates: &[((usize, usize), PredicateTemplate)],
data: &CodegenData<'_>,
) -> io::Result<String> {
let mut items = Vec::new();
for ((rule_index, pred_index), predicate) in predicates {
// The deprecated `ParserPredicate` table (SemIR is the active path). A
// `<fail=...>` wrapper on a non-constant-false predicate has no encoding
// in this legacy enum, so render the transparent inner template; the
// SemIR predicate builder carries the message on the active path.
let predicate = predicate_effective_template(predicate);
let expression = match predicate {
PredicateTemplate::True => "antlr4_runtime::ParserPredicate::True".to_owned(),
PredicateTemplate::Hook => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"hook predicates lower only through parser SemIR",
));
}
PredicateTemplate::False => "antlr4_runtime::ParserPredicate::False".to_owned(),
PredicateTemplate::FalseWithMessage { message } => {
format!(
"antlr4_runtime::ParserPredicate::FalseWithMessage {{ message: \"{}\" }}",
rust_string(message)
)
}
PredicateTemplate::Invoke { value } => {
format!("antlr4_runtime::ParserPredicate::Invoke {{ value: {value} }}")
}
PredicateTemplate::LocalIntEquals { value } => {
format!("antlr4_runtime::ParserPredicate::LocalIntEquals {{ value: {value} }}")
}
PredicateTemplate::LocalIntLessOrEqual { value } => {
format!("antlr4_runtime::ParserPredicate::LocalIntLessOrEqual {{ value: {value} }}")
}
PredicateTemplate::TextEquals(_) => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"TextEquals is only supported for lexer predicates",
));
}
PredicateTemplate::TokenStartColumnEquals(_)
| PredicateTemplate::ColumnLessThan(_)
| PredicateTemplate::ColumnGreaterOrEqual(_) => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"column predicates are only supported for lexer predicates",
));
}
PredicateTemplate::LookaheadTextEquals { offset, text } => {
format!(
"antlr4_runtime::ParserPredicate::LookaheadTextEquals {{ offset: {offset}, text: \"{}\" }}",
rust_string(text)
)
}
PredicateTemplate::LookaheadNotEquals { offset, token_name } => {
let token_type = token_type_for_name(data, token_name).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate token {token_name}"),
)
})?;
format!(
"antlr4_runtime::ParserPredicate::LookaheadNotEquals {{ offset: {offset}, token_type: {token_type} }}"
)
}
PredicateTemplate::TokenPairAdjacent => {
"antlr4_runtime::ParserPredicate::TokenPairAdjacent".to_owned()
}
PredicateTemplate::ContextChildRuleTextNotEquals { rule_name, text } => {
let rule_index = data
.rule_names
.iter()
.position(|name| name == rule_name)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate rule {rule_name}"),
)
})?;
format!(
"antlr4_runtime::ParserPredicate::ContextChildRuleTextNotEquals {{ rule_index: {rule_index}, text: \"{}\" }}",
rust_string(text)
)
}
// `predicate_effective_template` above already unwrapped any wrapper;
// the constructor never nests, so this is unreachable.
PredicateTemplate::WithFailMessage { .. } => {
unreachable!("predicate_effective_template unwraps the fail-message wrapper")
}
};
items.push(format!("({rule_index}, {pred_index}, {expression})"));
}
Ok(format!("[{}]", items.join(", ")))
}
/// Renders parser rule-argument metadata for generated calls into the runtime.
fn render_parser_rule_arg_array(args: &[(usize, usize, RuleArgTemplate)]) -> String {
let items = args
.iter()
.map(|(source_state, rule_index, value)| {
let (value, inherit_local) = match value {
RuleArgTemplate::Literal(value) => (*value, false),
RuleArgTemplate::InheritLocal => (0, true),
};
format!(
"antlr4_runtime::ParserRuleArg {{ source_state: {source_state}, rule_index: {rule_index}, value: {value}, inherit_local: {inherit_local} }}"
)
})
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders the generated parser base construction.
///
/// When a non-default unknown-predicate policy is configured, the constructor
/// installs it on the `BaseParser` so the generated recursive-descent path
/// (which evaluates predicates without going through `ParserRuntimeOptions`)
/// honors `--sem-unknown` too, rather than leaving the field at `AssumeTrue`.
fn render_parser_base_initialization(unknown_policy_literal: Option<&str>) -> String {
let needs_mut = unknown_policy_literal.is_some();
let mut out = if needs_mut {
" let mut base = BaseParser::with_semantic_hooks(input, data, hooks);".to_owned()
} else {
" let base = BaseParser::with_semantic_hooks(input, data, hooks);".to_owned()
};
if let Some(policy) = unknown_policy_literal {
write!(
out,
"\n base.set_unknown_predicate_policy({policy});"
)
.expect("writing to a string cannot fail");
}
out
}
/// Renders the parser-module convenience that wires text input through the
/// caller-selected lexer, token stream, parser, and entry rule in one call.
fn render_parser_parse_convenience(type_name: &str) -> String {
let output_type_name = format!("{type_name}ParseOutput");
format!(
r#"/// Result from [`parse_with_parser`].
///
/// Keeps the generated parser available after the entry rule runs so callers
/// can inspect diagnostics or recover the parser-owned token stream.
#[derive(Debug)]
pub struct {output_type_name}<R, L>
where
L: TokenSource,
{{
pub result: R,
pub parser: {type_name}<L>,
}}
/// Parses UTF-8 text by constructing the lexer, token stream, parser, and
/// caller-selected entry rule in one call.
///
/// Pass the generated lexer constructor and a parser entry rule, for example
/// `parse(src, MyGrammarLexer::new, {type_name}::file)`.
///
/// The returned [`antlr4_runtime::ParsedFile`] owns the canonical token store,
/// flat CST storage, and entry-rule root.
/// Use [`parse_with_parser`] instead when the caller also needs parser
/// diagnostics after the entry rule runs.
pub fn parse<L: TokenSource>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<antlr4_runtime::NodeId, antlr4_runtime::AntlrError>,
) -> Result<antlr4_runtime::ParsedFile, antlr4_runtime::AntlrError>
{{
let {output_type_name} {{ result, parser }} = parse_with_parser(input, lexer, entry)?;
Ok(parser.into_parsed_file(result))
}}
/// Parses UTF-8 text like [`parse`] while returning the parser after the entry
/// rule has run.
///
/// This keeps the compact generated setup path available for callers that also
/// need `Parser::number_of_syntax_errors()` or `{type_name}::into_token_stream()`.
pub fn parse_with_parser<L: TokenSource, R>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<R, antlr4_runtime::AntlrError>,
) -> Result<{output_type_name}<R, L>, antlr4_runtime::AntlrError>
{{
let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref()));
let tokens = CommonTokenStream::new(lexer);
let mut parser = {type_name}::new(tokens);
let result = entry(&mut parser)?;
Ok({output_type_name} {{ result, parser }})
}}"#
)
}
fn token_type_for_name(data: &CodegenData<'_>, token_name: &str) -> Option<usize> {
data.symbolic_names
.iter()
.position(|name| name.as_deref() == Some(token_name))
}
/// Converts an ANTLR token/rule name into an upper-snake Rust constant name.
fn rust_const_name(name: &str) -> String {
let words = split_identifier_words(name);
let ident = if words.is_empty() {
"TOKEN".to_owned()
} else {
ascii_uppercase(&words.join("_"))
};
sanitize_identifier(&ident)
}
/// Converts ASCII letters to upper case without using allocation-hiding string
/// case helpers disallowed by the strict Clippy policy.
fn ascii_uppercase(value: &str) -> String {
value.chars().map(|ch| ch.to_ascii_uppercase()).collect()
}
fn max_len(left: &[Option<String>], right: &[Option<String>]) -> usize {
left.len().max(right.len())
}
#[cfg(test)]
#[allow(clippy::disallowed_methods)] // `insta` assertion macros unwrap internal I/O.
mod tests {
use super::*;
use antlr4_runtime::atn::parser_atn::{ParserAtnBuilder, ParserTransitionSpec};
#[test]
fn renders_module_level_metadata_helpers() {
let rendered = render_metadata("TParser", &minimal_parser_data());
assert!(
rendered.contains("pub fn metadata() -> &'static GrammarMetadata {\n &METADATA\n}")
);
assert!(rendered.contains(
"pub fn rule_names() -> &'static [&'static str] {\n METADATA.rule_names()\n}"
));
}
#[test]
fn converts_names_to_rust_identifiers() {
assert_eq!(module_name("ExprLexer"), "expr_lexer");
assert_eq!(rust_function_name("sourceFile"), "source_file");
assert_eq!(rust_const_name("LPAREN"), "LPAREN");
assert_eq!(rust_const_name("Q_COLONCOLON"), "Q_COLONCOLON");
assert_eq!(rust_const_name("LineStrExprStart"), "LINE_STR_EXPR_START");
assert_eq!(rust_const_name("UnicodeClassLL"), "UNICODE_CLASS_LL");
assert_eq!(rust_function_name("gen"), "r#gen");
assert_eq!(rust_function_name("try"), "r#try");
assert_eq!(rust_function_name("Self"), "r#self");
assert!(is_rust_keyword("Self"));
}
#[test]
fn renders_parser_rustdoc_with_entry_rule_methods() {
let data = CodegenData {
rule_names: vec![
"sourceFile".to_owned(),
"declaration".to_owned(),
"script".to_owned(),
"try".to_owned(),
],
..CodegenData::default()
};
let entry_rule_indices = vec![0, 2];
let rendered = render_parser_rustdoc(
&parser_public_rule_method_names(&data.rule_names),
&entry_rule_indices,
);
assert!(rendered.contains("Likely parser entry-rule methods"));
assert!(rendered.contains("/// - `source_file()`"));
assert!(rendered.contains("/// - `script()`"));
assert!(rendered.contains("All parser rule methods:"));
assert!(rendered.contains("/// - `declaration()`"));
assert!(rendered.contains("/// - `r#try()`"));
assert!(rendered.contains("cannot"));
assert!(rendered.contains("semantic choice"));
}
#[test]
fn infers_entry_rule_candidates_from_rule_call_graph() {
let atn = entry_candidate_atn();
assert_eq!(
likely_parser_entry_rule_indices_from_atn(&atn, 4),
vec![0, 2, 3]
);
}
#[test]
fn generated_parser_rustdoc_is_attached_to_parser_type() {
let rendered = render_parser("DemoParser", &minimal_parser_data()).expect("parser renders");
assert!(rendered.contains(
"/// Generated parser. Each grammar rule is exposed as a public method.\n///\n/// Pick an entry-rule method"
));
assert!(rendered.contains(
"/// Likely parser entry-rule methods (not called by other rules):\n/// - `s()`"
));
assert!(rendered.contains(
"/// All parser rule methods:\n/// - `s()`\n#[derive(Debug)]\npub struct DemoParser<L, H = antlr4_runtime::NoSemanticHooks>"
));
}
#[test]
fn generated_parser_embeds_only_versioned_packed_atn_data() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(rendered.contains("static PARSER_ATN_DATA: &[u32]"));
assert!(rendered.contains("static ATN_CELL: OnceLock<ParserAtn>"));
assert!(rendered.contains("ParserAtn::from_static(PARSER_ATN_DATA)"));
assert!(rendered.contains("generated parser ATN is incompatible with this runtime"));
assert!(rendered.contains("pub fn parser_atn() -> &'static ParserAtn"));
assert!(rendered.contains("fn parser_atn() -> &'static ParserAtn"));
assert!(!rendered.contains("AtnDeserializer"));
assert!(!rendered.contains("SerializedAtn"));
}
#[test]
fn parser_rule_method_names_reserve_recognizer_reuse_accessors() {
let rule_names = vec![
"tokenStream".to_owned(),
"into_token_stream".to_owned(),
"token_stream_rule".to_owned(),
"reset".to_owned(),
"setTokenStream".to_owned(),
"clearDfa".to_owned(),
"addErrorListener".to_owned(),
"removeErrorListeners".to_owned(),
"regularRule".to_owned(),
];
assert_eq!(
parser_public_rule_method_names(&rule_names),
[
"token_stream_rule",
"into_token_stream_rule",
"token_stream_rule_2",
"reset_rule",
"set_token_stream_rule",
"clear_dfa_rule",
"add_error_listener_rule",
"remove_error_listeners_rule",
"regular_rule"
]
);
}
#[test]
fn generated_modules_start_with_file_level_header() {
let lexer = render_lexer(
"TLexer",
&predicate_lexer_data(),
false,
SemUnknownPolicy::default(),
&SemPatternFile::default(),
false,
)
.expect("lexer module should render");
let parser =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(GENERATED_MODULE_HEADER.contains(concat!(
"@generated by ",
env!("CARGO_PKG_NAME"),
" v",
env!("CARGO_PKG_VERSION")
)));
assert!(GENERATED_MODULE_HEADER.contains(env!("CARGO_PKG_REPOSITORY")));
for rendered in [&lexer, &parser] {
assert!(rendered.starts_with(GENERATED_MODULE_HEADER));
assert!(rendered[GENERATED_MODULE_HEADER.len()..].starts_with("use antlr4_runtime::"));
assert!(!rendered.contains("#!["));
assert!(rendered.contains(GENERATED_MODULE_FOOTER));
assert!(
rendered.ends_with("pub use self::__antlr4_rust_generated::*;\n"),
"generated module should end with exactly one trailing newline and no blank line at EOF"
);
}
}
fn compile_test_parser_rule(
atn: &ParserAtn,
rule_index: usize,
inline_action_states: &BTreeSet<usize>,
) -> Option<GeneratedParserRule> {
let decision_by_state = decision_by_state(atn);
let action_states = BTreeSet::new();
let generated_action_states = BTreeSet::new();
let predicate_coordinates = BTreeSet::new();
let generated_predicate_coordinates = BTreeSet::new();
let context = GeneratedParserCompileContext {
atn,
decision_by_state: &decision_by_state,
rule_args: &[],
inline_action_states,
action_states: &action_states,
generated_action_states: &generated_action_states,
predicate_coordinates: &predicate_coordinates,
generated_predicate_coordinates: &generated_predicate_coordinates,
};
compile_generated_parser_rule(&context, rule_index)
}
fn finish_atn(builder: ParserAtnBuilder) -> ParserAtn {
builder.finish().expect("valid packed parser ATN")
}
fn transition_atn(
make_transition: impl FnOnce(&mut ParserAtnBuilder) -> ParserTransitionSpec,
) -> ParserAtn {
let mut builder = ParserAtnBuilder::new(10);
for _ in 0..10 {
builder
.add_state(AtnStateKind::Basic, Some(0))
.expect("state");
}
builder
.set_rule_to_start_state(vec![0, 0, 0])
.expect("rule start states");
builder
.set_rule_to_stop_state(vec![9, 9, 9])
.expect("rule stop states");
let transition = make_transition(&mut builder);
builder
.add_transition(0, transition)
.expect("test transition");
finish_atn(builder)
}
fn only_transition(atn: &ParserAtn) -> ParserTransition<'_> {
atn.state(0)
.expect("transition source")
.transitions()
.first()
.expect("test transition")
}
fn mt(token_type: i32, follow_state: usize) -> GeneratedParserStep {
GeneratedParserStep::MatchToken {
token_type,
follow_state,
}
}
fn ms(intervals: Vec<(i32, i32)>, follow_state: usize) -> GeneratedParserStep {
GeneratedParserStep::MatchSet {
token_set: None,
intervals,
follow_state,
}
}
fn mts(
token_set: usize,
intervals: Vec<(i32, i32)>,
follow_state: usize,
) -> GeneratedParserStep {
GeneratedParserStep::MatchSet {
token_set: Some(token_set),
intervals,
follow_state,
}
}
fn mnts(
token_set: usize,
intervals: Vec<(i32, i32)>,
follow_state: usize,
) -> GeneratedParserStep {
GeneratedParserStep::MatchNotSet {
token_set: Some(token_set),
intervals,
follow_state,
}
}
fn mns(intervals: Vec<(i32, i32)>, follow_state: usize) -> GeneratedParserStep {
GeneratedParserStep::MatchNotSet {
token_set: None,
intervals,
follow_state,
}
}
fn cr(rule_index: usize) -> GeneratedParserStep {
GeneratedParserStep::CallRule {
source_state: 100 + rule_index,
rule_index,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}
}
fn adaptive_loop(decision: usize) -> GeneratedParserStep {
GeneratedParserStep::StarLoop {
state: 1_000 + decision,
decision,
enter_alt: 1,
exit_alt: 2,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
plus_loop: false,
fast_path: None,
body: vec![mt(2, 0)],
}
}
fn expensive_ladder_rule(rule_index: usize, next: Option<usize>) -> GeneratedParserRule {
let mut steps = Vec::new();
if let Some(next) = next {
steps.push(cr(next));
}
steps.push(adaptive_loop(rule_index * 2));
steps.push(adaptive_loop(rule_index * 2 + 1));
if next.is_none() {
steps.push(mt(1, 0));
}
test_rule(rule_index, steps)
}
fn test_rule(rule_index: usize, steps: Vec<GeneratedParserStep>) -> GeneratedParserRule {
GeneratedParserRule {
rule_index,
entry_state: rule_index * 2,
left_recursive: false,
steps,
}
}
#[test]
fn compiles_linear_parser_rule_body() {
let atn = linear_rule_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("linear rule should compile");
assert_eq!(body.rule_index, 0);
assert_eq!(body.entry_state, 0);
assert_eq!(body.steps, [mt(1, 2), mt(TOKEN_EOF, 3)]);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
assert!(rendered.contains("match_token_recovering(1, 2, atn())"));
assert!(rendered.contains("generated_diagnostics_checkpoint()"));
assert!(rendered.contains("restore_generated_diagnostics(__generated_diagnostic_marker)"));
}
#[test]
fn compiles_block_decision_with_adaptive_prediction() {
let atn = block_decision_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("block decision rule should compile");
assert_eq!(
body.steps,
[GeneratedParserStep::Decision {
state: 1,
decision: 0,
track_alt_number: true,
allow_semantic_context: false,
force_context: false,
fast_path: Some(GeneratedDecisionFastPath {
arms: vec![
GeneratedDecisionFastArm {
alt: 1,
intervals: vec![(1, 1)],
},
GeneratedDecisionFastArm {
alt: 2,
intervals: vec![(2, 2)],
},
],
}),
alts: vec![vec![mt(1, 4)], vec![mt(2, 4)]],
}]
);
let rendered =
render_generated_rule_dispatch(&[Some(body.clone())], &[], &BTreeMap::new(), false);
assert!(rendered.contains("parse_generated_rule_0"));
assert!(rendered.contains("sync_decision(atn(), 1, !__ctx.has_matched_child(), false)"));
assert!(rendered.contains("ll1_decision_prediction(atn(), 1)"));
// Stage 1 is the SLL probe (no LL loop on the empty-context conflict);
// stage 2 re-runs with the real context only when full context is needed.
assert!(rendered.contains("adaptive_predict_stream_info_sll_probe(0, 0"));
assert!(rendered.contains("adaptive_predict_stream_info_with_context(0, 0"));
assert!(rendered.contains(
"intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()))"
));
assert!(!rendered.contains("self.base.prediction_context(atn())"));
let rendered_with_alt_numbers =
render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), true);
assert!(rendered_with_alt_numbers.contains("__ctx.set_alt_number(1);"));
assert!(rendered_with_alt_numbers.contains("__ctx.set_alt_number(2);"));
}
#[test]
fn compiles_star_loop_with_adaptive_prediction() {
let atn = star_loop_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("star loop rule should compile");
assert_eq!(
body.steps,
[GeneratedParserStep::StarLoop {
state: 1,
decision: 0,
enter_alt: 1,
exit_alt: 2,
track_alt_number: true,
allow_semantic_context: false,
force_context: false,
plus_loop: false,
fast_path: None,
body: vec![mt(1, 4)],
}]
);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
assert!(rendered.contains("loop {"));
// A `*` loop starts NOT iterated: its first sync is at the loop entry
// (single-token deletion), so the iteration flag inits to `false`.
assert!(rendered.contains("let mut __loop_iter_1 = false;"));
assert!(
rendered.contains("sync_decision(atn(), 1, !__ctx.has_matched_child(), __loop_iter_1)")
);
assert!(rendered.contains("__loop_iter_1 = true;"));
assert!(rendered.contains("1 => {"));
assert!(rendered.contains("2 => {"));
assert!(rendered.contains("break;"));
assert!(rendered.contains("ll1_decision_prediction(atn(), 1)"));
assert!(rendered.contains("adaptive_predict_stream_info_sll_probe(0, 0"));
assert!(rendered.contains("adaptive_predict_stream_info_with_context(0, 0"));
}
#[test]
fn compiles_plus_loop_back_with_adaptive_prediction() {
let atn = plus_loop_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("plus loop rule should compile");
assert_eq!(
body.steps,
[
mt(1, 3),
GeneratedParserStep::StarLoop {
state: 4,
decision: 0,
enter_alt: 1,
exit_alt: 2,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
plus_loop: true,
fast_path: None,
body: vec![mt(1, 3)],
}
]
);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
// A `+` loop's mandatory first element is iteration 1, so the iteration
// flag inits to `true`: its first loop-back sync recovers with multi-token
// `consumeUntil`, matching ANTLR's PLUS_LOOP_BACK.
assert!(rendered.contains("let mut __loop_iter_4 = true;"));
assert!(
rendered.contains("sync_decision(atn(), 4, !__ctx.has_matched_child(), __loop_iter_4)")
);
}
#[test]
fn compiles_plus_block_body_decision_with_adaptive_prediction() {
let atn = plus_block_decision_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("plus block decision rule should compile");
let body_decision = GeneratedParserStep::Decision {
state: 1,
decision: 0,
track_alt_number: true,
allow_semantic_context: false,
force_context: false,
fast_path: Some(GeneratedDecisionFastPath {
arms: vec![
GeneratedDecisionFastArm {
alt: 1,
intervals: vec![(1, 1)],
},
GeneratedDecisionFastArm {
alt: 2,
intervals: vec![(2, 2)],
},
],
}),
alts: vec![vec![mt(1, 4)], vec![mt(2, 4)]],
};
assert_eq!(
body.steps,
[
body_decision.clone(),
GeneratedParserStep::StarLoop {
state: 5,
decision: 1,
enter_alt: 1,
exit_alt: 2,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
plus_loop: true,
fast_path: None,
body: vec![body_decision],
}
]
);
}
#[test]
fn compiles_left_recursive_parser_rule() {
let atn = left_recursive_rule_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("left-recursive rule should compile");
assert!(body.left_recursive);
assert_eq!(body.rule_index, 0);
assert_eq!(body.entry_state, 0);
assert_eq!(
body.steps,
[
mt(1, 2),
GeneratedParserStep::LeftRecursiveLoop {
state: 2,
decision: 0,
enter_alt: 1,
exit_alt: 2,
rule_index: 0,
entry_state: 0,
body: vec![GeneratedParserStep::Decision {
state: 3,
decision: 1,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: vec![vec![
GeneratedParserStep::Precedence(2),
mt(2, 10),
GeneratedParserStep::CallRule {
source_state: 10,
rule_index: 0,
precedence: GeneratedRuleCallPrecedence::Literal(3),
},
]],
}],
}
]
);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
assert!(rendered.contains("parse_generated_rule_0_precedence(precedence, allow_fallback)"));
assert!(
rendered.contains("push_new_recursion_context_with_previous(0isize, 0, &mut __ctx)")
);
assert!(rendered.contains("parse_rule_precedence_from_generated(0, 3)"));
assert!(rendered.contains("precpred(_ctx, 2)"));
assert!(
rendered.contains(
"let __prediction = match self.base.left_recursive_loop_enter_prediction(atn(), 2, __precedence)"
)
);
assert!(rendered.contains("Some(true) => antlr4_runtime::ParserAtnPrediction"));
assert!(
rendered
.contains("adaptive_predict_stream_info_with_context(0, __prediction_precedence")
);
assert!(rendered.contains(
"Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt { .. }) => antlr4_runtime::ParserAtnPrediction { alt: 2, requires_full_context: true, has_semantic_context: false, diagnostic: None }"
));
}
#[test]
fn drops_generated_rules_that_call_disabled_rules() {
let mut rules = vec![
Some(GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
}),
None,
Some(GeneratedParserRule {
rule_index: 2,
entry_state: 10,
left_recursive: false,
steps: vec![mt(1, 0)],
}),
];
drop_rules_calling_disabled_rules(&mut rules);
assert!(rules[0].is_none());
assert!(rules[1].is_none());
assert!(rules[2].is_some());
}
#[test]
fn generated_parent_keeps_interpreted_child_call() {
let rules = vec![
Some(GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
}),
None,
];
let rendered =
render_generated_rule_dispatch(&rules, &[true, false], &BTreeMap::new(), false);
assert!(rendered.contains(
"0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("parse_generated_rule_1_dispatch"));
}
#[test]
fn classifies_expensive_long_leading_call_chains_as_atn_preferred() {
let mut rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
assert_eq!(
generated_atn_preferred_rule_calls(&rules, &[]),
vec![true; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN]
);
let required = BTreeSet::from([ATN_PREFERRED_LEADING_CALL_CHAIN_MIN - 1]);
assert_eq!(
generated_atn_preferred_rule_calls_excluding(
&rules,
&[],
&generated_rule_callers_reaching(&rules, &required),
),
vec![false; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN],
"portable-local owners and generated callers must stay on the generated path"
);
rules.truncate(ATN_PREFERRED_LEADING_CALL_CHAIN_MIN - 1);
assert_eq!(
generated_atn_preferred_rule_calls(&rules, &[]),
vec![false; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN - 1]
);
}
#[test]
fn graph_reachability_traverses_backwards_from_every_target() {
let mut graph = DiGraph::new();
let nodes = (0..6)
.map(|value| graph.add_node(value))
.collect::<Vec<_>>();
graph.add_edge(nodes[0], nodes[1], ());
graph.add_edge(nodes[1], nodes[2], ());
graph.add_edge(nodes[3], nodes[4], ());
assert_eq!(
graph_nodes_reaching(&graph, &BTreeSet::from([2, 4, 99])),
BTreeSet::from([0, 1, 2, 3, 4, 99])
);
}
#[test]
fn atn_preferred_rule_calls_reject_simple_operator_ladders() {
let simple_rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let steps = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
vec![adaptive_loop(rule_index), mt(1, 0)]
} else {
vec![cr(rule_index + 1), adaptive_loop(rule_index)]
};
Some(test_rule(rule_index, steps))
})
.collect::<Vec<_>>();
assert_eq!(
generated_atn_preferred_rule_calls(&simple_rules, &[]),
vec![false; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN]
);
let expensive_rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
assert_eq!(
generated_atn_preferred_rule_calls(&expensive_rules, &[]),
vec![true; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN]
);
}
#[test]
fn atn_preferred_rule_calls_propagate_through_expensive_wrappers() {
let mut rules = Vec::new();
rules.push(Some(test_rule(
0,
vec![mt(9, 0), adaptive_loop(100), adaptive_loop(101), cr(1)],
)));
rules.push(Some(test_rule(
1,
vec![mt(8, 0), adaptive_loop(102), adaptive_loop(103), cr(2)],
)));
for rule_index in 2..(2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN) {
let next = if rule_index + 1 == 2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
rules.push(Some(expensive_ladder_rule(rule_index, next)));
}
rules.push(Some(test_rule(10, vec![cr(2)])));
let mut expected = vec![true; 2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN];
expected.push(false);
assert_eq!(generated_atn_preferred_rule_calls(&rules, &[]), expected);
}
#[test]
fn renders_atn_preferred_generated_child_calls_as_interpreted_by_default() {
let rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
let direct_generated_rule_calls = vec![true; rules.len()];
let rule_names = Vec::new();
let rendered = render_generated_rule_dispatch_with_rule_names(
&rules,
&direct_generated_rule_calls,
&rule_names,
&BTreeMap::new(),
true,
false,
None,
None,
);
// ATN-preferred children route through `parse_rule_precedence_from_generated`:
// the rule's generated dispatch arm is `generated_only()`-guarded, so in normal
// mode the wrapper parses the child interpreted (optimization preserved) while
// buffering its actions in position (correct ordering).
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(1, 0)"));
}
#[test]
fn renders_atn_preferred_dispatch_only_for_generated_only_mode() {
let mut rules = Vec::new();
rules.push(Some(test_rule(
0,
vec![mt(9, 0), adaptive_loop(100), adaptive_loop(101), cr(2)],
)));
rules.push(Some(test_rule(1, vec![mt(1, 0)])));
for rule_index in 2..(2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN) {
let next = if rule_index + 1 == 2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
rules.push(Some(expensive_ladder_rule(rule_index, next)));
}
let direct_generated_rule_calls = vec![true; rules.len()];
let rule_names = Vec::new();
let rendered = render_generated_rule_dispatch_with_rule_names(
&rules,
&direct_generated_rule_calls,
&rule_names,
&BTreeMap::new(),
true,
false,
None,
None,
);
assert!(rendered.contains(
"0 if self.generated_only() => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
assert!(!rendered.contains(
"0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
// The ATN-preferred child call routes through the buffering wrapper.
assert!(rendered.contains("self.parse_rule_precedence_from_generated(2, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(2, 0)"));
}
#[test]
fn embedded_rules_never_use_atn_preferred_fallback() {
let rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
let direct_generated_rule_calls = vec![true; rules.len()];
let adaptive_decisions = BTreeSet::new();
let ll1_decision_arms = BTreeMap::new();
let predicates = BTreeMap::new();
let rule_has_attrs = vec![false; rules.len()];
let init_entry = BTreeMap::new();
let after = BTreeMap::new();
let call_args = BTreeMap::new();
let rule_arg0 = vec![None; rules.len()];
let rendered = render_generated_rule_dispatch_with_rule_names(
&rules,
&direct_generated_rule_calls,
&[],
&BTreeMap::new(),
true,
false,
Some(EmbeddedStepRender {
force_adaptive: false,
adaptive_decisions: &adaptive_decisions,
ll1_decision_arms: &ll1_decision_arms,
predicates: &predicates,
rule_has_attrs: &rule_has_attrs,
init_entry: &init_entry,
after: &after,
call_args: &call_args,
rule_arg0: &rule_arg0,
}),
None,
);
assert!(!rendered.contains("if self.generated_only()"));
assert!(rendered.contains(
"0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
assert!(rendered.contains("self.parse_generated_rule_1_dispatch(0, false)"));
assert!(!rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
}
#[test]
fn compiles_token_set_transitions() {
let range_atn = transition_atn(|_| ParserTransitionSpec::Range {
target: 7,
start: 2,
stop: 4,
});
let range = only_transition(&range_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
range,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(ms(vec![(2, 4)], 7)), 7))
);
let set_atn = transition_atn(|builder| {
let set = builder.add_interval_set([(1, 1), (5, 6)]).expect("set");
ParserTransitionSpec::Set { target: 8, set }
});
let set_transition = only_transition(&set_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
set_transition,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(ms(vec![(1, 1), (5, 6)], 8)), 8))
);
let not_set_atn = transition_atn(|builder| {
let set = builder.add_interval_set([(1, 1)]).expect("set");
ParserTransitionSpec::NotSet { target: 9, set }
});
let not_set_transition = only_transition(¬_set_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
not_set_transition,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(mns(vec![(1, 1)], 9)), 9))
);
let dense_ranges = (1..=256)
.step_by(2)
.map(|token| (token, token))
.collect::<Vec<_>>();
let dense_set_atn = transition_atn(|builder| {
let set = builder
.add_interval_set(dense_ranges.iter().copied())
.expect("dense set");
ParserTransitionSpec::Set { target: 8, set }
});
let dense_set_transition = only_transition(&dense_set_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
dense_set_transition,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(mts(0, dense_ranges, 8)), 8))
);
}
#[test]
fn compiles_generated_action_transitions_only_for_allowed_states() {
let action_atn = transition_atn(|_| ParserTransitionSpec::Action {
target: 8,
rule_index: 2,
action_index: Some(0),
context_dependent: false,
});
let action = only_transition(&action_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
None
);
let mut generated_action_states = BTreeSet::new();
generated_action_states.insert(4);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &BTreeSet::new(),
generated: &generated_action_states,
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((
Some(GeneratedParserStep::Action {
source_state: 4,
rule_index: 2,
}),
8
))
);
}
#[test]
fn compiles_rule_call_precedence_from_rule_args() {
let rule_atn = transition_atn(|_| ParserTransitionSpec::Rule {
target: 1,
rule_index: 2,
follow_state: 8,
precedence: 0,
});
let rule = only_transition(&rule_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[(4, 2, RuleArgTemplate::Literal(6))],
rule,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((
Some(GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 2,
precedence: GeneratedRuleCallPrecedence::Literal(6),
}),
8
))
);
assert_eq!(
compile_generated_parser_transition(
4,
&[(4, 2, RuleArgTemplate::InheritLocal)],
rule,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((
Some(GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 2,
precedence: GeneratedRuleCallPrecedence::InheritLocal,
}),
8
))
);
}
#[test]
fn parses_boolean_literal_rule_arguments() {
let data = parser_fixture_data("boolean-rule-arguments/T.g4");
let args = structural_parser_rule_args(&data)
.expect("structural rule-call arguments should resolve")
.into_iter()
.map(|(_, rule_index, value)| (rule_index, value))
.collect::<Vec<_>>();
assert_eq!(
args,
[
(1, RuleArgTemplate::Literal(1)),
(1, RuleArgTemplate::Literal(0)),
]
);
}
#[test]
fn compiles_synthetic_noop_action_transitions_as_epsilon() {
let action_atn = transition_atn(|_| ParserTransitionSpec::Action {
target: 8,
rule_index: 2,
action_index: None,
context_dependent: false,
});
let action = only_transition(&action_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((None, 8))
);
}
#[test]
fn rejects_known_non_inline_noop_action_transitions() {
let action_atn = transition_atn(|_| ParserTransitionSpec::Action {
target: 8,
rule_index: 2,
action_index: None,
context_dependent: false,
});
let action = only_transition(&action_atn);
let mut action_states = BTreeSet::new();
action_states.insert(4);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &action_states,
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
None
);
}
#[test]
fn compiles_parser_predicates_as_viable_when_no_metadata_is_active() {
let predicate_atn = transition_atn(|_| ParserTransitionSpec::Predicate {
target: 8,
rule_index: 2,
pred_index: 1,
context_dependent: false,
});
let predicate = only_transition(&predicate_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
predicate,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((None, 8))
);
}
#[test]
fn compiles_generated_parser_predicate_transitions() {
let predicate_atn = transition_atn(|_| ParserTransitionSpec::Predicate {
target: 8,
rule_index: 2,
pred_index: 1,
context_dependent: false,
});
let predicate = only_transition(&predicate_atn);
let mut predicates = BTreeSet::new();
predicates.insert((2, 1));
let generated_predicates = predicates.clone();
assert_eq!(
compile_generated_parser_transition(
4,
&[],
predicate,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &predicates,
generated: &generated_predicates,
}
),
Some((
Some(GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 1,
}),
8
))
);
}
#[test]
fn renders_fail_option_parser_predicate_error() {
let mut rendered = String::new();
render_generated_step(
&mut rendered,
&GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 1,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(
rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 2, 1, &__ctx, __precedence)"
)
);
assert!(rendered.contains("failed_predicate_option_error(2, __message)"));
assert!(rendered.contains("failed_predicate_error(\"semantic predicate\")"));
}
fn render_call_rule_step(
direct_generated_rule_calls: &[bool],
atn_preferred_rule_calls: &[bool],
) -> String {
let mut rendered = String::new();
render_generated_step(
&mut rendered,
&GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
},
2,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls,
atn_preferred_rule_calls,
},
);
rendered
}
#[test]
fn atn_preferred_child_with_after_action_routes_through_dispatch_wrapper() {
// An ATN-preferred child that carries an `@after` action
// (direct_generated_rule_calls[1] == false) must go through
// `parse_rule_precedence_from_generated`, which preserves interpreted routing
// (the rule's generated dispatch arm is guarded by `generated_only()`) while
// BUFFERING the child's body actions and `@after` in position.
let rendered = render_call_rule_step(&[true, false], &[true, true]);
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(1, 0)"));
}
#[test]
fn atn_preferred_child_without_after_also_routes_through_dispatch_wrapper() {
// An ATN-preferred child WITHOUT `@after` (direct_generated_rule_calls[1] ==
// true) must ALSO route through `parse_rule_precedence_from_generated`, not the
// bare `parse_interpreted_rule_precedence`: the bare interpreted call runs the
// child's body actions immediately, which reorders them before the generated
// parent's buffered actions. The wrapper buffers them in position instead while
// still parsing the child interpreted (the dispatch arm is `generated_only()`
// guarded).
let rendered = render_call_rule_step(&[true, true], &[true, true]);
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(1, 0)"));
}
#[test]
fn rejects_known_parser_predicates_without_generated_metadata() {
let predicate_atn = transition_atn(|_| ParserTransitionSpec::Predicate {
target: 8,
rule_index: 2,
pred_index: 1,
context_dependent: false,
});
let predicate = only_transition(&predicate_atn);
let mut predicates = BTreeSet::new();
predicates.insert((2, 1));
assert_eq!(
compile_generated_parser_transition(
4,
&[],
predicate,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &predicates,
generated: &BTreeSet::new(),
}
),
None
);
}
#[test]
fn parse_rule_fallback_runs_parser_actions() {
let fallback = render_parser_parse_rule_fallback(false, false, &[], true, false, None);
assert!(fallback.contains(
"parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence"
));
assert!(fallback.contains("for action in actions { self.run_action(action, tree); }"));
assert!(fallback.contains("Ok(tree)"));
}
#[test]
fn parser_action_dispatch_falls_back_to_semantic_hook() {
let method = render_parser_action_method(true, &BTreeSet::new());
assert!(method.contains("fn run_action"));
assert!(method.contains("self.base.parser_action_hook(action, tree)"));
}
#[test]
fn parser_action_assume_override_gets_explicit_noop_arm() {
// An `assume-*` action override drops its translated arm, but must NOT
// fall through to the `parser_action_hook` catch-all — that would fail
// loud under the Error policy (NoSemanticHooks) or run a user side
// effect for a coordinate the manifest reports as a no-op fallback. It
// gets an explicit empty arm instead. A `hook`/`error` override is not
// in this set, so it still falls through to the hook.
let mut assume_noop = BTreeSet::new();
assume_noop.insert(7_usize);
let method = render_parser_action_method(true, &assume_noop);
// The assume-* state has its own empty arm, placed before the catch-all.
let noop_at = method
.find("7 => {}")
.expect("assume-* action state gets an explicit no-op arm");
let hook_at = method
.find("self.base.parser_action_hook(action, tree)")
.expect("the hook catch-all is still emitted for hook/unknown states");
assert!(
noop_at < hook_at,
"the assume-* no-op arm must precede the hook catch-all"
);
}
#[test]
fn parser_st_actions_do_not_emit_replay_machinery() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
for removed in [
format!("{}{}", "Generated", "Action"),
format!("{}{}", "generated", "_actions"),
format!("{}{}", "Member", "Snapshot"),
format!("{}{}", "run_after", "_actions"),
format!("{}{}", "int_members", "_checkpoint"),
format!("{}{}", "restore_int", "_members"),
format!("{}{}", "CTX_ROOTED", "_ACTION_STATES"),
] {
assert!(
!rendered.contains(&removed),
"removed parser replay machinery leaked into generated module: {removed}"
);
}
}
#[test]
fn embedded_init_action_runs_at_rule_entry() {
let data = parser_fixture_data("embedded-init/T.g4");
let rendered = render_parser_with_options(
"TParser",
&data,
ParserRenderOptions {
embedded: true,
..ParserRenderOptions::default()
},
)
.expect("embedded parser should render");
let start_at = rendered
.find("let __rule_start")
.expect("generated rule captures rule start");
let init_at = rendered
.find("println!(\"init\");")
.expect("embedded @init body is emitted");
let body_at = rendered
.find("let mut __consumed_eof")
.expect("generated rule body follows entry setup");
assert!(
start_at < init_at && init_at < body_at,
"embedded @init must run after rule entry setup and before the rule body"
);
}
#[test]
fn embedded_left_recursive_actions_resolve_deleted_leading_labels() {
let data = parser_fixture_data("left-recursive-labels/T.g4");
let model =
structural_embedded_model(&data, false).expect("structural model should resolve");
insta::assert_debug_snapshot!("left_recursive_label_alternatives", model.rules[1].alts);
let embedded =
build_embedded_parser_data(&data, "TParser", "T", ParserRenderOptions::default())
.expect("embedded actions should resolve deleted left-recursive labels");
insta::assert_debug_snapshot!("left_recursive_label_actions", embedded.inline_actions);
}
#[test]
fn embedded_listener_forwards_error_nodes() {
let rendered = render_parser_with_options(
"TParser",
&minimal_parser_data(),
ParserRenderOptions {
embedded: true,
..ParserRenderOptions::default()
},
)
.expect("embedded parser should render");
assert!(rendered.contains("ErrorNodeView as RuntimeErrorNode"));
assert!(rendered.contains("pub struct ErrorNode<'a>"));
assert!(
rendered.contains("fn visit_error_node(&mut self, _node: &ErrorNode) -> Result<(), E>")
);
assert!(rendered.contains("listener.visit_error_node(&ErrorNode::new("));
}
#[test]
fn embedded_active_context_preserves_invocation_states() {
let rendered = render_parser_with_options(
"TParser",
&parser_fixture_data("left-recursive-labels/T.g4"),
ParserRenderOptions {
embedded: true,
..ParserRenderOptions::default()
},
)
.expect("embedded parser should render");
assert!(rendered.contains("invocation_states: Vec<isize>"));
assert!(rendered.contains("__invocation_states: invocation_states"));
assert!(rendered.contains("::__from_child_node(node, &self.__invocation_states)"));
assert!(rendered.contains("::__from_listener_node(context, invocation_states.as_deref())"));
assert!(rendered.contains("pub fn walk_with_invocation_states"));
assert!(
!rendered.contains("__GeneratedRuleContext::Active { .. } => Vec::new()"),
"active contexts must preserve their invoking-state chain"
);
}
#[test]
fn context_surface_names_disambiguate_normalized_label_collisions() {
let data = parser_fixture_data("context-name-collision/T.g4");
let model =
structural_embedded_model(&data, false).expect("structural model should resolve");
let names = context_surface_names(&model);
let every_rule = model
.rules
.iter()
.position(|rule| rule.name == "everyRule")
.expect("everyRule fixture rule");
assert_ne!(names.rules[every_rule].listener_method, "every_rule");
let stored_tree = model
.rules
.iter()
.position(|rule| rule.name == "storedTree")
.expect("storedTree fixture rule");
assert_ne!(names.rules[stored_tree].context_type, "StoredTreeContext");
insta::assert_debug_snapshot!("context_surface_name_collision", names);
}
#[test]
fn typed_context_accessors_are_cardinality_aware_and_rust_shaped() {
let rendered = render_parser(
"TParser",
&parser_fixture_data("left-recursive-labels/T.g4"),
)
.expect("parser should render");
let s_context = rendered
.split_once("impl<'a, State> SContext<'a, State> {")
.expect("s context impl")
.1
.split_once("impl<State> std::fmt::Display for SContext")
.expect("s context display impl")
.0;
assert!(s_context.contains("pub fn e(&self) -> Result<EContext<'a>, MissingChildError>"));
assert!(!s_context.contains("pub fn s(&self"));
assert!(!s_context.contains("_all(&self)"));
let e_context = rendered
.split_once("impl<'a, State> EContext<'a, State> {")
.expect("e context impl")
.1
.split_once("impl<State> std::fmt::Display for EContext")
.expect("e context display impl")
.0;
assert!(
e_context.contains("pub fn e_children(&self) -> impl Iterator<Item = EContext<'a>>")
);
assert!(e_context.contains("pub fn int_token(&self) -> Option<TerminalNode<'a>>"));
assert!(e_context.contains("pub fn star_token(&self) -> Option<TerminalNode<'a>>"));
assert!(e_context.contains("pub fn plus_token(&self) -> Option<TerminalNode<'a>>"));
assert!(e_context.contains("pub fn left(&self) -> Option<EContext<'a>>"));
assert!(e_context.contains("pub fn right(&self) -> Option<EContext<'a>>"));
assert!(!e_context.contains("pub fn s(&self"));
assert!(!e_context.contains("pub fn INT(&self"));
assert!(!e_context.contains("_all(&self)"));
}
#[test]
fn literal_labels_keep_terminal_action_semantics() {
let data = parser_fixture_data("typed-tree-walkers/Calculator.g4");
let model =
structural_embedded_model(&data, false).expect("structural model should resolve");
let labeled_tokens = model
.rules
.iter()
.find(|rule| rule.name == "labeledTokens")
.expect("labeledTokens rule");
let literal = labeled_tokens
.alts
.iter()
.flat_map(|alternative| &alternative.refs)
.find(|element| element.label.as_deref() == Some("literal"))
.expect("literal label");
assert!(literal.is_block);
assert_eq!(literal.token_types.len(), 1);
}
#[test]
fn structural_set_token_types_expand_literal_ranges() {
let data = parser_fixture_data("typed-tree-walkers/Calculator.g4");
let vocabulary = &data
.semantic
.expect("semantic grammar")
.recognizer
.vocabulary;
let mut literals = vocabulary
.by_literal
.iter()
.map(|(literal, token_type)| (literal.clone(), *token_type))
.collect::<Vec<_>>();
literals.sort_unstable_by_key(|(_, token_type)| *token_type);
assert!(literals.len() >= 3, "fixture needs three literal tokens");
let (start, start_type) = literals[0].clone();
let (stop, stop_type) = literals[2].clone();
let range = SetElement::Range {
source: grammar::model::ElementId::new(0),
start,
stop,
span: SourceSpan::empty(grammar::frontend::SourceId::new(0)),
options: Vec::new(),
};
let expected = (start_type..=stop_type).collect::<Vec<_>>();
assert_eq!(
structural_set_token_types(false, std::slice::from_ref(&range), vocabulary),
expected
);
assert_eq!(
structural_set_token_types(true, &[range], vocabulary),
(1..=vocabulary.max_token_type())
.filter(|token_type| !expected.contains(token_type))
.collect::<Vec<_>>()
);
}
#[test]
fn typed_context_accessors_preserve_ebnf_list_and_single_labels() {
let data = parser_fixture_data("combined-contexts/Shapes.g4");
let model =
structural_embedded_model(&data, false).expect("structural model should resolve");
let start = model
.rules
.iter()
.find(|rule| rule.name == "start")
.expect("start rule");
let many = start
.alts
.iter()
.find(|alternative| alternative.label.as_deref() == Some("Many"))
.expect("many alternative");
let rest = many
.refs
.iter()
.filter(|element| element.label.as_deref() == Some("rest"))
.collect::<Vec<_>>();
assert_eq!(rest.len(), 2);
assert!(rest.iter().all(|element| element.stable_accessor));
assert_eq!(rest[0].cardinality, embedded::ChildCardinality::ONE);
assert_eq!(
rest[1].cardinality,
embedded::ChildCardinality { min: 0, max: None }
);
let rendered = render_parser("ShapesParser", &data).expect("parser should render");
let many_context = rendered
.split_once("impl<'a, State> ManyLabelContext<'a, State> {")
.expect("many context impl")
.1
.split_once("impl<State> std::fmt::Display for ManyLabelContext")
.expect("many context display impl")
.0;
assert!(
many_context.contains("pub fn rest(&self) -> impl Iterator<Item = AtomContext<'a>>")
);
let latest_context = rendered
.split_once("impl<'a, State> LatestContext<'a, State> {")
.expect("latest context impl")
.1
.split_once("impl<State> std::fmt::Display for LatestContext")
.expect("latest context display impl")
.0;
assert!(
latest_context
.contains("pub fn value(&self) -> Result<AtomContext<'a>, MissingChildError>")
);
assert!(latest_context.contains(".skip(0).last()"));
}
#[test]
fn non_embedded_parser_action_disables_generated_rule() {
let rendered =
render_parser("TParser", &action_parser_data()).expect("parser should render");
assert!(
!rendered.contains("parse_generated_rule_0_dispatch"),
"non-embedded parser action rules must stay on the interpreted path"
);
assert!(rendered.contains("self.base.parser_action_hook(action, tree)"));
assert!(!rendered.contains(&format!("{}{}", "Generated", "Action")));
assert!(!rendered.contains(&format!("{}{}", "generated", "_actions")));
}
#[test]
fn context_superclass_does_not_disable_generated_rules() {
let data = parser_fixture_data("context-superclass/T.g4");
let rendered = render_parser("TParser", &data).expect("parser should render");
assert!(rendered.contains("parse_generated_rule_0"));
assert!(rendered.contains("track_alt_numbers: true"));
}
#[test]
fn generated_parser_handles_diagnostic_reporting() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(!rendered.contains("if !self.base.report_diagnostic_errors() || __generated_only"));
assert!(
rendered.contains("self.parse_interpreted_rule_precedence(rule_index, precedence)?")
);
}
#[test]
fn generated_only_mode_disables_missing_rule_fallback() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(rendered.contains("ANTLR4_RUST_GENERATED_ONLY"));
assert!(rendered.contains("let __generated_only = self.generated_only();"));
assert!(!rendered.contains("GeneratedRuleError::Recoverable"));
assert!(rendered.contains("generated parser did not emit rule {}"));
}
#[test]
fn require_generated_parser_reports_missing_rules() {
let error = require_all_parser_rules_generated(&[None], &minimal_parser_data())
.expect_err("missing generated rule should fail strict mode");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(
error.to_string(),
"generated parser did not emit 1 rule(s): s"
);
}
#[test]
fn portable_local_semantics_reject_missing_generated_owner() {
let error = require_portable_local_rules_generated(
&[None],
&BTreeSet::from([0]),
&minimal_parser_data(),
)
.expect_err("portable local semantics cannot use interpreted fallback");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(
error.to_string(),
"portable local semantics require 1 generated parser rule(s): s"
);
}
#[test]
fn portable_local_semantics_reject_missing_generated_caller() {
let required = atn_rule_callers_reaching(&entry_candidate_atn(), &BTreeSet::from([1]), 4);
assert_eq!(required, BTreeSet::from([0, 1, 2]));
let rules = vec![
None,
Some(test_rule(1, Vec::new())),
Some(test_rule(2, Vec::new())),
Some(test_rule(3, Vec::new())),
];
let data = CodegenData {
rule_names: vec![
"firstEntry".to_owned(),
"child".to_owned(),
"secondEntry".to_owned(),
"recursive".to_owned(),
],
..CodegenData::default()
};
let error = require_portable_local_rules_generated(&rules, &required, &data)
.expect_err("interpreted callers cannot bypass generated local state");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(
error.to_string(),
"portable local semantics require 1 generated parser rule(s): firstEntry"
);
}
#[test]
fn renders_parse_convenience_without_replacing_manual_constructor() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(rendered.contains("pub struct TParserParseOutput<R, L>"));
assert!(rendered.contains("pub result: R,"));
assert!(rendered.contains("pub parser: TParser<L>,"));
assert!(rendered.contains("pub fn parse<L: TokenSource>("));
assert!(rendered.contains("pub fn parse_with_parser<L: TokenSource, R>("));
assert!(
!rendered
.contains(") -> Result<R, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource,")
);
assert!(!rendered.contains(
") -> Result<TParserParseOutput<R, L>, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource,"
));
assert!(rendered.contains("lexer: impl FnOnce(antlr4_runtime::InputStream) -> L"));
assert!(rendered.contains("antlr4_runtime::InputStream::new(input.as_ref())"));
assert!(rendered.contains("let tokens = CommonTokenStream::new(lexer);"));
assert!(rendered.contains("let result = entry(&mut parser)?;"));
assert!(rendered.contains("Ok(TParserParseOutput { result, parser })"));
assert!(rendered.contains(
"let TParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?;"
));
assert!(rendered.contains("Ok(parser.into_parsed_file(result))"));
assert!(rendered.contains("pub fn new(input: CommonTokenStream<L>) -> Self"));
assert!(
rendered.contains("pub fn with_hooks(input: CommonTokenStream<L>, hooks: H) -> Self")
);
}
#[test]
fn generated_parse_output_name_does_not_collide_with_parser_type() {
let rendered =
render_parser("ParseOutput", &minimal_parser_data()).expect("parser should render");
assert!(rendered.contains("pub struct ParseOutputParseOutput<R, L>"));
assert!(rendered.contains("pub parser: ParseOutput<L>,"));
assert!(
rendered
.contains(") -> Result<ParseOutputParseOutput<R, L>, antlr4_runtime::AntlrError>")
);
assert!(rendered.contains("Ok(ParseOutputParseOutput { result, parser })"));
}
#[test]
fn generated_parser_reports_lexer_errors_on_outer_success() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(rendered.contains("if allow_generated_fallback {"));
assert!(rendered.contains("self.base.report_generated_parser_diagnostics();"));
assert!(rendered.contains("fn number_of_syntax_errors(&self) -> usize"));
assert!(!rendered.contains("self.base.report_token_source_errors();"));
}
#[test]
fn generated_parser_exposes_owned_token_stream() {
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
assert!(rendered.contains("pub const fn token_stream(&self) -> &CommonTokenStream<L>"));
assert!(rendered.contains("self.base.token_stream()"));
assert!(
rendered
.contains("pub const fn token_stream_mut(&mut self) -> &mut CommonTokenStream<L>")
);
assert!(rendered.contains("self.base.token_stream_mut()"));
assert!(
rendered.contains("pub fn set_token_stream(&mut self, input: CommonTokenStream<L>)")
);
assert!(rendered.contains("self.base.set_token_stream(input)"));
assert!(rendered.contains("pub fn reset(&mut self)"));
assert!(rendered.contains("self.base.reset()"));
assert!(rendered.contains("pub fn clear_dfa(&mut self)"));
assert!(rendered.contains("simulator.clear_dfa()"));
assert!(rendered.contains("ParserAtnSimulator::clear_shared_dfa(atn())"));
assert!(rendered.contains("pub fn add_error_listener<T>(&mut self, listener: T)"));
assert!(rendered.contains(
"T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,"
));
assert!(rendered.contains("self.base.add_error_listener(listener)"));
assert!(rendered.contains("pub fn remove_error_listeners(&mut self)"));
assert!(rendered.contains("self.base.remove_error_listeners()"));
assert!(rendered.contains("pub fn into_token_stream(self) -> CommonTokenStream<L>"));
assert!(rendered.contains("self.base.into_token_stream()"));
assert!(
rendered.contains("pub const fn token_store(&self) -> &antlr4_runtime::TokenStore")
);
assert!(rendered.contains("self.base.token_store()"));
assert!(rendered.contains("pub fn into_token_store(self) -> antlr4_runtime::TokenStore"));
assert!(rendered.contains("self.base.into_token_store()"));
}
#[test]
fn generated_parser_renames_rule_wrapper_that_collides_with_token_stream_accessor() {
let mut data = minimal_parser_data();
data.rule_names = vec!["tokenStream".to_owned()];
let rendered = render_parser("TParser", &data).expect("parser should render");
assert!(rendered.contains("pub const fn token_stream(&self) -> &CommonTokenStream<L>"));
assert!(rendered.contains(
"pub fn token_stream_rule(&mut self) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError>"
));
assert!(rendered.contains("/// - `token_stream_rule()`"));
assert!(!rendered.contains("/// - `token_stream()`"));
assert!(!rendered.contains("pub fn token_stream(&mut self)"));
}
#[test]
fn generated_rule_recovers_own_sync_failure_unless_top_level() {
// A rule's own sync failure (`__sync_error`) is fatal only at the top-level
// public entry (`allow_fallback`); a nested child recovers it locally and
// returns a partial subtree (so a parent never recovers the child's failure
// on the parent context, losing the child subtree). Assert the generated
// catch arm gates the `Fatal` return on `allow_fallback` and otherwise runs
// `recover_generated_rule` + `finish_rule` + `Ok`.
let rendered =
render_parser("TParser", &minimal_parser_data()).expect("parser should render");
let sync_arm = rendered
.find("if let Some(__error) = __sync_error {")
.expect("sync-error catch arm present");
let rest = &rendered[sync_arm..];
// Inside the sync arm, the Fatal return is guarded by `if allow_fallback`.
let guard = rest
.find("if allow_fallback {")
.expect("fatal return gated on allow_fallback");
let fatal = rest
.find("return Err(GeneratedRuleError::Fatal(__error));")
.expect("fatal return present");
assert!(
guard < fatal,
"Fatal return must be inside the allow_fallback guard"
);
let count = rest
.find("self.base.record_generated_syntax_error();")
.expect("fatal sync path records syntax error");
assert!(
guard < count && count < fatal,
"fatal sync path must increment before returning"
);
// And the nested-child path recovers locally and returns Ok.
let recover = rest
.find("self.base.recover_generated_rule(&mut __ctx, atn(), __error);")
.expect("local recovery present in sync arm");
assert!(
recover > guard,
"recover path follows the guarded fatal return"
);
assert!(rest[recover..].contains("return Ok(__tree);"));
}
#[test]
fn call_rule_step_skips_child_action_scaffolding_without_parser_actions() {
let rendered = render_call_rule_step(&[true, true], &[false, false]);
assert!(rendered.contains("let __child = self.parse_generated_rule_1_dispatch(0, false).map_err(GeneratedRuleError::into_error);"));
assert!(rendered.contains("self.base.discard_invoking_state(__invoking_marker);"));
assert!(rendered.contains("let __child = __child?;"));
assert!(rendered.contains("self.base.add_parse_child(&mut __ctx, __child);"));
assert!(!rendered.contains("__child_action_marker"));
assert!(!rendered.contains("__child_member_checkpoint"));
assert!(!rendered.contains(&format!(
"{}{}{}{}",
"Generated", "Action::", "Member", "Snapshot"
)));
assert!(!rendered.contains(&format!(
"{}{}{}",
"CTX_ROOTED", "_ACTION_STATES", ".contains"
)));
}
#[test]
fn renders_wildcard_match_through_recovering_path() {
// A wildcard (`.`) must go through the recovering match (modeled as an
// empty-complement not-set over the full vocabulary) so a wildcard at EOF
// performs ANTLR's single-token insertion instead of aborting the rule.
let rule = GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![GeneratedParserStep::MatchWildcard { follow_state: 7 }],
};
let rendered = render_generated_rule_dispatch(&[Some(rule)], &[], &BTreeMap::new(), false);
// Recovering not-set over 1..=max with an empty exclusion = "any token",
// threading the wildcard's follow state for EOF-insertion follow checks.
assert!(
rendered.contains("match_not_set_recovering(&[], 1, atn().max_token_type(), 7, atn())")
);
assert!(rendered.contains("__consumed_eof |= __match.consumed_eof();"));
// The old non-recovering call must be gone.
assert!(!rendered.contains("self.base.match_wildcard()"));
}
#[test]
fn renders_packed_token_sets_for_generated_matches_and_lookahead() {
let step = mts(4, vec![(2, 4), (9, 9)], 7);
let rule = GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![step.clone()],
};
let rendered = render_generated_rule_dispatch(&[Some(rule)], &[], &BTreeMap::new(), false);
assert!(rendered.contains(
"match_token_set_recovering(atn().token_set(4).expect(\"generated parser token-set index\"), 7, atn())"
));
assert_eq!(
leading_lookahead_condition(&[step], "__la"),
Some(
"atn().token_set(4).expect(\"generated parser token-set index\").contains(__la)"
.to_owned()
)
);
let not_step = mnts(5, vec![(3, 6)], 8);
let not_rule = GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![not_step.clone()],
};
let rendered =
render_generated_rule_dispatch(&[Some(not_rule)], &[], &BTreeMap::new(), false);
assert!(rendered.contains(
"match_not_token_set_recovering(atn().token_set(5).expect(\"generated parser token-set index\"), 1, atn().max_token_type(), 8, atn())"
));
assert_eq!(
leading_lookahead_condition(&[not_step], "__la"),
Some(
"(1..=atn().max_token_type()).contains(&__la) && !(atn().token_set(5).expect(\"generated parser token-set index\").contains(__la))"
.to_owned()
)
);
}
#[test]
fn generated_decision_does_not_reject_semantic_context_metadata() {
let alts = vec![vec![mt(1, 0)], vec![]];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("ll1_decision_prediction(atn(), 1)"));
assert!(rendered.contains("prediction_mode() != antlr4_runtime::PredictionMode::Sll"));
assert!(!rendered.contains("has_semantic_context"));
}
#[test]
fn generated_decision_filters_semantic_predicate_alts() {
let alts = vec![
vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(1, 2),
],
vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 1,
},
mt(1, 3),
],
vec![mt(2, 4)],
];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("if __prediction.has_semantic_context"));
assert!(rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 1, 0, &__ctx, __precedence)"
));
assert!(rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 1, 1, &__ctx, __precedence)"
));
assert!(rendered.contains("__semantic_la == 1"));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: __alt, ..__prediction }")
);
assert!(rendered.contains("no_viable_alternative_error(__decision_start)"));
assert!(!rendered.contains("__sync_error = Some(__error.clone())"));
}
#[test]
fn generated_decision_does_not_hoist_portable_predicate_past_local_action() {
let alts = vec![
vec![
GeneratedParserStep::Action {
source_state: 5,
rule_index: 1,
},
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(1, 2),
],
vec![mt(1, 3)],
];
let declarations = vec![
Vec::new(),
vec!["let mut __antlr_local_seen = false;".to_owned()],
];
let inline_actions = BTreeMap::from([(5, "__antlr_local_seen = true;".to_owned())]);
let predicates = BTreeMap::from([((1, 0), ("__antlr_local_seen".to_owned(), None))]);
let required_generated_rules = BTreeSet::from([1]);
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: Some(PortableLocalStepRender {
declarations: &declarations,
inline_actions: &inline_actions,
predicates: &predicates,
required_generated_rules: &required_generated_rules,
}),
inline_action_statements: &inline_actions,
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(!rendered.contains("let __semantic_alt"));
assert!(!rendered.contains("__semantic_la == 1 && (__antlr_local_seen)"));
let assignment = rendered
.find("__antlr_local_seen = true;")
.expect("portable assignment is rendered in the committed alternative");
let predicate = rendered
.find("if !(__antlr_local_seen)")
.expect("portable predicate is evaluated in the committed alternative");
assert!(assignment < predicate);
}
#[test]
fn generated_decision_records_adaptive_diagnostics() {
let alts = vec![vec![mt(1, 4)], vec![mt(2, 5)]];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 16,
decision: 0,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(
rendered.contains("record_generated_prediction_diagnostic(atn(), 16, &__prediction)")
);
assert!(!rendered.contains("__diagnostic_la"));
}
#[test]
fn generated_semantic_decision_reports_filtered_ambiguity_diagnostics() {
let alts = vec![
vec![mt(2, 4)],
vec![mt(2, 5)],
vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(2, 6),
],
];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 16,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("if self.base.report_diagnostic_errors()"));
assert!(rendered.contains("let __diagnostic_la = self.base.la(1);"));
assert!(rendered.contains("if __diagnostic_la == 2"));
assert!(rendered.contains("__diagnostic_alts.push(1);"));
assert!(rendered.contains("__diagnostic_alts.push(2);"));
assert!(rendered.contains(
"record_generated_ambiguity_diagnostic(atn(), 16, __decision_start, __decision_start, &__diagnostic_alts)"
));
}
#[test]
fn generated_loop_filters_failed_leading_predicate_to_exit_alt() {
let body = vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(3, 4),
];
let mut rendered = String::new();
render_generated_star_loop(
&mut rendered,
StarLoopRender {
state: 1,
decision: 0,
alts: (1, 2),
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
plus_loop: false,
fast_path: None,
body: &body,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("if __prediction.alt == 1"));
assert!(rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 1, 0, &__ctx, __precedence)"
));
assert!(rendered.contains("__semantic_la == 3"));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: 2, ..__prediction }")
);
}
#[test]
fn generated_loop_filters_portable_local_predicate() {
let body = vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(3, 4),
];
let declarations = vec![vec!["let mut __antlr_local_seen = false;".to_owned()]];
let predicates = BTreeMap::from([((1, 0), ("__antlr_local_seen".to_owned(), None))]);
let required_generated_rules = BTreeSet::from([1]);
let mut rendered = String::new();
render_generated_star_loop(
&mut rendered,
StarLoopRender {
state: 1,
decision: 0,
alts: (1, 2),
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
plus_loop: false,
fast_path: None,
body: &body,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: Some(PortableLocalStepRender {
declarations: &declarations,
inline_actions: &BTreeMap::new(),
predicates: &predicates,
required_generated_rules: &required_generated_rules,
}),
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("__semantic_la == 3 && (__antlr_local_seen)"));
assert!(!rendered.contains("parser_semantic_ir_predicate_matches"));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: 2, ..__prediction }")
);
}
#[test]
fn generated_loop_filters_first_nested_predicated_decision() {
let body = vec![GeneratedParserStep::Decision {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: vec![
vec![mt(1, 4)],
vec![mt(3, 4)],
vec![
GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 0,
},
mt(2, 4),
],
],
}];
let mut rendered = String::new();
render_generated_star_loop(
&mut rendered,
StarLoopRender {
state: 1,
decision: 1,
alts: (1, 2),
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
plus_loop: false,
fast_path: None,
body: &body,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
track_context_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("(__semantic_la == 1) || (__semantic_la == 3)"));
// The lookahead guard comes first so `&&` short-circuits on it before the
// predicate hook runs; a non-matching lookahead must not trigger a
// fail-loud hit for an unknown/hook predicate on a non-candidate alt.
assert!(rendered.contains(
"(__semantic_la == 2 && self.base.parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 2, 0, &__ctx, __precedence))"
));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: 2, ..__prediction }")
);
}
#[test]
fn semantic_candidate_condition_guards_predicate_behind_lookahead() {
// A leading predicate followed by a token match must render as
// `lookahead && predicate`, so `&&` short-circuits on the side-effect-free
// lookahead before invoking the predicate hook. Otherwise an alternative
// whose first token cannot match the current lookahead would still
// evaluate its hook/unknown predicate, recording a spurious fail-loud
// `Unsupported` hit under `--sem-unknown=hook`/`error` and rejecting a
// later syntactically viable alternative.
let steps = vec![
GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 0,
},
mt(7, 4),
];
let condition = semantic_alt_candidate_condition(&steps, None, None);
let la_at = condition
.find("__semantic_la == 7")
.expect("condition includes the leading lookahead guard");
let pred_at = condition
.find("parser_semantic_ir_predicate_matches_with_context_and_local")
.expect("condition includes the leading predicate");
assert!(
la_at < pred_at,
"lookahead must be evaluated before the predicate hook: {condition}"
);
assert!(
condition.starts_with("__semantic_la == 7 &&"),
"the lookahead guard must be the first `&&` operand: {condition}"
);
}
#[test]
fn semantic_alt_guard_classifies_unresolved_rule_call_alt() {
// An alt whose first consuming step is a rule call, with no leading
// predicate or lookahead, is "unresolved" (FIRST set not computed here).
let rule_call = vec![GeneratedParserStep::CallRule {
source_state: 5,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}];
assert!(semantic_alt_guard_is_unresolved(&rule_call, None));
// A token-led alt is resolved (concrete lookahead), so NOT unresolved.
assert!(!semantic_alt_guard_is_unresolved(&[mt(7, 4)], None));
// A predicate-led alt is resolved (guarded by the predicate).
assert!(!semantic_alt_guard_is_unresolved(
&[GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 0,
}],
None,
));
// A pure epsilon alt (no consuming step) legitimately matches; not unresolved.
assert!(!semantic_alt_guard_is_unresolved(&[], None));
}
#[test]
fn semantic_alt_search_orders_unresolved_alts_last() {
// `{p()}? 'a' | x | 'a'` (alt 2 = rule call `x`): when alt 1's predicate
// fails, the two-pass search tries resolved-guard alts first (alt 3's
// concrete lookahead), then the unresolved rule-call alt as a last
// resort. So alt 3 is NOT shadowed by alt 2, AND alt 2 stays reachable.
let alts = vec![
vec![
GeneratedParserStep::Predicate {
rule_index: 0,
pred_index: 0,
},
mt(1, 4),
],
vec![GeneratedParserStep::CallRule {
source_state: 5,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
vec![mt(1, 4)],
];
let alt_conditions = alts
.iter()
.map(|steps| semantic_alt_candidate_condition(steps, None, None))
.collect::<Vec<_>>();
let mut rendered = String::new();
render_semantic_alt_search(&mut rendered, "", &alt_conditions, &alts, None);
// The resolved token alt 3 is emitted before the unresolved rule-call
// alt 2 (which keeps its real `true` condition as a last-resort branch),
// so alt 3 wins on a matching lookahead but alt 2 is still reachable.
let alt3_at = rendered
.find("Some(3)")
.expect("resolved token alt 3 is present");
let alt2_at = rendered
.find("Some(2)")
.expect("unresolved rule-call alt 2 is still present (last resort)");
assert!(
alt3_at < alt2_at,
"resolved alt 3 must be tried before the unresolved rule-call alt 2: {rendered}"
);
// Alt 2 is NOT disabled (`if false`); it keeps a reachable branch.
assert!(
!rendered.contains("if false {"),
"unresolved alt must be a last-resort branch, not disabled: {rendered}"
);
assert!(
rendered.contains("if __semantic_la == 1 {\n Some(3)"),
"the token alt keeps its concrete lookahead guard: {rendered}"
);
}
#[test]
fn semantic_alt_search_keeps_lone_unresolved_alt_reachable() {
// `{p()}? 'a' | x` (alt 2 = rule call `x`, the only non-predicated alt):
// when alt 1's predicate fails and no resolved alt matches, the search
// must still try the unresolved alt rather than emitting nothing (which
// would be a spurious NoViableAlt). Codex's counter-example.
let alts = vec![
vec![
GeneratedParserStep::Predicate {
rule_index: 0,
pred_index: 0,
},
mt(1, 4),
],
vec![GeneratedParserStep::CallRule {
source_state: 5,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
];
let alt_conditions = alts
.iter()
.map(|steps| semantic_alt_candidate_condition(steps, None, None))
.collect::<Vec<_>>();
let mut rendered = String::new();
render_semantic_alt_search(&mut rendered, "", &alt_conditions, &alts, None);
// The lone unresolved alt is reachable via its real condition, not disabled.
assert!(
rendered.contains("Some(2)") && !rendered.contains("if false {"),
"a lone unresolved alt must remain reachable as a last resort: {rendered}"
);
}
#[test]
fn parses_column_predicate_templates() {
assert_eq!(
parse_predicate_template(r#"<TokenStartColumnEquals("0")>"#),
Some(PredicateTemplate::TokenStartColumnEquals(0))
);
assert_eq!(
parse_predicate_template(r#"<Column()> \< 2"#),
Some(PredicateTemplate::ColumnLessThan(2))
);
assert_eq!(
parse_predicate_template("<Column()> >= 2"),
Some(PredicateTemplate::ColumnGreaterOrEqual(2))
);
}
#[test]
fn native_comparison_predicate_falls_through_instead_of_aborting() {
// A native target-language comparison predicate merely contains a `<`
// operator; it is not an ANTLR `<...>` StringTemplate, so it must fall
// through to the unknown-predicate policy (no template) rather than
// aborting codegen with "unsupported target predicate template".
let native = parser_fixture_data("native-comparison/T.g4");
let templates = structural_predicate_templates(
&native,
SemanticsKind::ParserPredicate,
&SemPatternFile::default(),
)
.expect("native comparison predicate must not abort generation");
assert!(
templates.is_empty(),
"native `<` comparison yields no template, deferring to policy: {templates:?}"
);
// A genuine untranslated `<...>` StringTemplate still errors.
let unsupported = parser_fixture_data("unsupported-predicate-template/T.g4");
let error = structural_predicate_templates(
&unsupported,
SemanticsKind::ParserPredicate,
&SemPatternFile::default(),
)
.expect_err("an untranslated <...> StringTemplate predicate must still abort");
assert!(
error
.to_string()
.contains("unsupported target predicate template")
);
}
#[test]
fn parses_predicate_fail_option_message() {
let data = parser_fixture_data("predicate-fail/T.g4");
let predicate = structural_predicates(&data)
.expect("structural predicate should resolve")
.into_iter()
.next()
.expect("fixture has one predicate");
assert_eq!(predicate.fail, Some("custom message".to_owned()));
assert_eq!(
predicate_template_with_fail_message(
PredicateTemplate::False,
"custom message".to_owned(),
),
PredicateTemplate::FalseWithMessage {
message: "custom message".to_owned()
}
);
// A non-constant-false predicate (hook, member, lookahead, …) preserves
// its `<fail=...>` message via the transparent `WithFailMessage` wrapper
// rather than discarding it.
let wrapped =
predicate_template_with_fail_message(PredicateTemplate::Hook, "hook failed".to_owned());
assert_eq!(
wrapped,
PredicateTemplate::WithFailMessage {
inner: Box::new(PredicateTemplate::Hook),
message: "hook failed".to_owned(),
}
);
// The wrapper is transparent to evaluation and generatability, and it
// exposes the message.
assert_eq!(
predicate_effective_template(&wrapped),
&PredicateTemplate::Hook
);
assert!(can_generate_parser_predicate(&wrapped));
assert_eq!(
predicate_template_fail_message(&wrapped),
Some("hook failed")
);
// Disposition follows the inner: a wrapped hook is still `Hooked`.
assert_eq!(
predicate_template_disposition(Some(&wrapped), SemUnknownPolicy::AssumeTrue),
SemanticsDisposition::Hooked
);
// A later `<fail=...>` replaces the message rather than nesting wrappers.
let rewrapped = predicate_template_with_fail_message(wrapped, "again".to_owned());
assert_eq!(
rewrapped,
PredicateTemplate::WithFailMessage {
inner: Box::new(PredicateTemplate::Hook),
message: "again".to_owned(),
}
);
}
#[test]
fn parses_supported_predicate_helpers() {
assert_eq!(
parse_invoke_predicate(r#"True():Invoke_pred()"#),
Some(PredicateTemplate::Invoke { value: true })
);
assert_eq!(
parse_invoke_predicate(r#"False():Invoke_pred()"#),
Some(PredicateTemplate::Invoke { value: false })
);
assert_eq!(
parse_predicate_template(r#"ParserPropertyCall({$parser}, "Property()")"#),
Some(PredicateTemplate::True)
);
assert_eq!(
parse_predicate_template("true"),
Some(PredicateTemplate::True)
);
assert_eq!(
parse_predicate_template("0==0"),
Some(PredicateTemplate::True)
);
assert_eq!(
parse_predicate_template("0 != 0"),
Some(PredicateTemplate::False)
);
assert_eq!(
parse_val_equals_predicate(r#"ValEquals("$i","2")"#),
Some(PredicateTemplate::LocalIntEquals { value: 2 })
);
assert_eq!(
parse_raw_local_int_less_or_equal_predicate("5 >= $_p"),
Some(PredicateTemplate::LocalIntLessOrEqual { value: 5 })
);
assert_eq!(
parse_predicate_template("this.IsRightArrow()"),
Some(PredicateTemplate::TokenPairAdjacent)
);
assert_eq!(
parse_predicate_template("this.IsLocalVariableDeclaration()"),
Some(PredicateTemplate::ContextChildRuleTextNotEquals {
rule_name: "local_variable_type".to_owned(),
text: "var".to_owned(),
})
);
}
#[test]
fn maps_kotlin_rcurl_java_action_to_lexer_pop_mode() {
for body in [
"popMode()",
"popMode();",
"this.popMode()",
"this.popMode();",
"if (!_modeStack.isEmpty()) { popMode(); }",
"if (!this._modeStack.isEmpty()) { popMode(); }",
"if (!_modeStack.isEmpty()) popMode()",
"if (!this._modeStack.isEmpty()) popMode();",
] {
assert_eq!(
parse_lexer_pop_mode_action(body),
Some(ActionTemplate::LexerPopMode),
"{body}"
);
}
let action = parse_lexer_pop_mode_action("if (!_modeStack.isEmpty()) { popMode(); }")
.expect("Kotlin RCURL action should lower");
let method = render_lexer_action_method(&[((1, 0), action)]);
assert!(method.contains("fn run_action"));
assert!(method.contains("_base.pop_mode();"));
}
#[test]
fn embedded_lexer_semantics_are_translated_without_template_classification() {
let data = lexer_fixture_data("embedded-lexer-semantics/L.g4");
let entries = collect_lexer_semantics(
&data,
true,
false,
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("embedded Rust bodies bypass portable template classification");
insta::assert_debug_snapshot!("embedded_lexer_semantics", entries);
}
#[test]
fn unsupported_lexer_action_renders_todo_marker() {
let data = lexer_fixture_data("unsupported-lexer-action/L.g4");
let actions = structural_lexer_action_templates(&data, &SemPatternFile::default())
.expect("structural lexer action should resolve")
.into_iter()
.map(|(_, action)| action)
.collect::<Vec<_>>();
assert_eq!(
actions,
[ActionTemplate::UnsupportedLexerAction {
rule_name: "ID".to_owned(),
body: "customJava();".to_owned(),
}]
);
assert_eq!(
render_lexer_action_statement(&actions[0]),
"/* TODO unsupported embedded lexer action in rule ID: {customJava();}; rewrite target-specific actions as portable lexer commands where possible */"
);
let method = render_lexer_action_method(&[((1, 0), actions[0].clone())]);
assert!(method.contains("TODO unsupported embedded lexer action in rule ID"));
assert!(!method.contains("fn run_action"));
assert_eq!(rust_block_comment_text("a */ b"), "a * / b");
let error = reject_unsupported_lexer_action_templates(&actions, false).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert!(error.to_string().contains(
"unsupported embedded lexer action in rule ID: {customJava();}; \
rewrite target-specific actions as portable lexer commands where possible"
));
reject_unsupported_lexer_action_templates(&actions, true)
.expect("unsupported-only lexer actions should be allowed in compatibility mode");
}
#[test]
fn mixed_supported_and_unsupported_lexer_actions_fail_even_when_allowed() {
let actions = vec![
ActionTemplate::UnsupportedLexerAction {
rule_name: "ID".to_owned(),
body: "setType(Foo);".to_owned(),
},
ActionTemplate::LexerPopMode,
];
let error = reject_unsupported_lexer_action_templates(&actions, true).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert!(error.to_string().contains(
"unsupported embedded lexer action in rule ID: {setType(Foo);}; \
rewrite target-specific actions as portable lexer commands where possible"
));
}
#[test]
fn lexer_action_diagnostic_summary_truncates_on_char_boundary() {
let body = format!("{}\u{00e9} tail", "a".repeat(95));
let summary = one_line_action_body(&body);
assert_eq!(summary, format!("{}...", "a".repeat(95)));
}
fn linear_rule_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
3
);
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(
1,
ParserTransitionSpec::Atom {
target: 2,
label: 1,
},
)
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 3,
label: TOKEN_EOF,
},
)
.expect("transition");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![3])
.expect("rule stop states");
finish_atn(atn)
}
fn block_decision_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::BlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
5
);
atn.set_end_state(1, 4).expect("block end state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 4,
label: 1,
},
)
.expect("transition");
atn.add_transition(
3,
ParserTransitionSpec::Atom {
target: 4,
label: 2,
},
)
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_decision_state(1).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![5])
.expect("rule stop states");
finish_atn(atn)
}
fn star_loop_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopEntry, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopBack, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
5
);
atn.set_loop_back_state(3, 4).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 4,
label: 1,
},
)
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_decision_state(1).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![5])
.expect("rule stop states");
finish_atn(atn)
}
fn plus_loop_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::PlusBlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::PlusLoopBack, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
6
);
atn.set_end_state(1, 3).expect("block end state");
atn.set_loop_back_state(5, 4).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 3,
label: 1,
},
)
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 })
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 })
.expect("transition");
atn.add_decision_state(4).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![6])
.expect("rule stop states");
finish_atn(atn)
}
fn plus_block_decision_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::PlusBlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::PlusLoopBack, Some(0))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
6
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
7
);
atn.set_end_state(1, 4).expect("block end state");
atn.set_loop_back_state(6, 5).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 4,
label: 1,
},
)
.expect("transition");
atn.add_transition(
3,
ParserTransitionSpec::Atom {
target: 4,
label: 2,
},
)
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 })
.expect("transition");
atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 })
.expect("transition");
atn.add_decision_state(1).expect("decision state");
atn.add_decision_state(5).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![7])
.expect("rule stop states");
finish_atn(atn)
}
fn left_recursive_rule_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopEntry, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::StarBlockStart, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
6
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
7
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopBack, Some(0))
.expect("state")
.index(),
8
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
9
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
10
);
atn.set_left_recursive_rule(0)
.expect("left-recursive rule start");
atn.set_precedence_rule_decision(2)
.expect("precedence decision");
atn.set_end_state(3, 6).expect("block end state");
atn.set_loop_back_state(7, 8).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(
1,
ParserTransitionSpec::Atom {
target: 2,
label: 1,
},
)
.expect("transition");
atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 7 })
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 })
.expect("transition");
atn.add_transition(
4,
ParserTransitionSpec::Precedence {
target: 5,
precedence: 2,
},
)
.expect("transition");
atn.add_transition(
5,
ParserTransitionSpec::Atom {
target: 10,
label: 2,
},
)
.expect("transition");
atn.add_transition(
10,
ParserTransitionSpec::Rule {
target: 0,
rule_index: 0,
follow_state: 6,
precedence: 3,
},
)
.expect("transition");
atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 8 })
.expect("transition");
atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 9 })
.expect("transition");
atn.add_decision_state(2).expect("decision state");
atn.add_decision_state(3).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![9])
.expect("rule stop states");
finish_atn(atn)
}
fn entry_candidate_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(1))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(1))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(2))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(2))
.expect("state")
.index(),
6
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(2))
.expect("state")
.index(),
7
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(3))
.expect("state")
.index(),
8
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(3))
.expect("state")
.index(),
9
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(3))
.expect("state")
.index(),
10
);
atn.add_transition(
0,
ParserTransitionSpec::Rule {
target: 3,
rule_index: 1,
follow_state: 2,
precedence: 0,
},
)
.expect("transition");
atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 })
.expect("transition");
atn.add_transition(
5,
ParserTransitionSpec::Rule {
target: 3,
rule_index: 1,
follow_state: 7,
precedence: 0,
},
)
.expect("transition");
atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 6 })
.expect("transition");
atn.add_transition(
8,
ParserTransitionSpec::Rule {
target: 8,
rule_index: 3,
follow_state: 10,
precedence: 0,
},
)
.expect("transition");
atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 9 })
.expect("transition");
atn.set_rule_to_start_state(vec![0, 3, 5, 8])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![1, 4, 6, 9])
.expect("rule stop states");
finish_atn(atn)
}
fn compile_test_fixture(relative: &str) -> &'static grammar::compiler::Compilation {
let root = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/antlr4-rust-gen")
.join(relative);
let compilation = grammar::compiler::compile(LoadOptions {
roots: vec![root],
library_directories: Vec::new(),
})
.unwrap_or_else(|error| panic!("test fixture {relative} should compile: {error:?}"));
Box::leak(Box::new(compilation))
}
fn parser_fixture_data(relative: &str) -> CodegenData<'static> {
let compilation = compile_test_fixture(relative);
let root = compilation
.roots
.first()
.expect("test grammar has one root");
let parser = compilation
.parser(root.parser.expect("test grammar has a parser"))
.expect("test parser artifact exists");
CodegenData::from_parser(parser, &compilation.sources)
}
fn lexer_fixture_data(relative: &str) -> CodegenData<'static> {
let compilation = compile_test_fixture(relative);
let root = compilation
.roots
.first()
.expect("test grammar has one root");
let lexer = compilation
.lexer(root.lexer.expect("test grammar has a lexer"))
.expect("test lexer artifact exists");
CodegenData::from_lexer(lexer, &compilation.sources)
}
fn minimal_parser_data() -> CodegenData<'static> {
parser_fixture_data("minimal/T.g4")
}
fn action_parser_data() -> CodegenData<'static> {
parser_fixture_data("parser-action/T.g4")
}
fn predicate_parser_data() -> CodegenData<'static> {
parser_fixture_data("predicate-parser/S.g4")
}
fn translated_predicate_parser_data() -> CodegenData<'static> {
parser_fixture_data("translated-predicate/S.g4")
}
fn predicate_lexer_data() -> CodegenData<'static> {
lexer_fixture_data("predicate-lexer/S.g4")
}
fn portable_bool_parser_data() -> CodegenData<'static> {
parser_fixture_data("portable-bool/S.g4")
}
#[test]
fn sem_unknown_flag_values_parse() {
assert_eq!(
SemUnknownPolicy::parse_flag("error").expect("error parses"),
SemUnknownPolicy::Error
);
assert_eq!(
SemUnknownPolicy::parse_flag("assume-true").expect("assume-true parses"),
SemUnknownPolicy::AssumeTrue
);
assert_eq!(
SemUnknownPolicy::parse_flag("assume-false").expect("assume-false parses"),
SemUnknownPolicy::AssumeFalse
);
assert_eq!(
SemUnknownPolicy::parse_flag("hook").expect("hook parses"),
SemUnknownPolicy::Hook
);
assert!(SemUnknownPolicy::parse_flag("bogus").is_err());
}
#[test]
fn set_lexer_predicate_template_replaces_or_appends() {
// A per-coordinate override must WIN over a built-in translation, so
// setting a covered coordinate replaces its template rather than adding
// a duplicate arm; an uncovered coordinate is appended.
let mut predicates = vec![((0, 0), PredicateTemplate::True)];
set_lexer_predicate_template(&mut predicates, (0, 0), PredicateTemplate::False);
assert_eq!(
predicates,
[((0, 0), PredicateTemplate::False)],
"replaces covered"
);
set_lexer_predicate_template(&mut predicates, (1, 2), PredicateTemplate::True);
assert_eq!(
predicates,
[
((0, 0), PredicateTemplate::False),
((1, 2), PredicateTemplate::True)
],
"appends uncovered"
);
}
#[test]
fn parser_action_assume_override_gets_noop_disposition() {
// Assumed parser actions get explicit no-op arms; hook/error overrides
// keep falling through to the parser action hook.
let data = predicate_parser_data(); // rule 0 = "s"
let mut action_state_rules = BTreeMap::new();
action_state_rules.insert(4_usize, 0_usize); // action state 4 belongs to rule `s`
for dispose in ["assume-true", "assume-false"] {
let patterns = parse_sem_patterns(&format!(
"version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\ndispose = \"{dispose}\"\n"
))
.expect("pattern file parses");
assert!(
parser_action_assume_overridden(&patterns, &data, &action_state_rules, 4),
"dispose {dispose}: an action state in rule `s` is assumed"
);
}
let hook_patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
assert!(
!parser_action_assume_overridden(&hook_patterns, &data, &action_state_rules, 4),
"hook overrides should keep routing through the hook arm"
);
assert!(
!parser_action_assume_overridden(
&SemPatternFile::default(),
&data,
&action_state_rules,
4
),
"no override -> concrete arm is kept"
);
}
#[test]
fn sem_pattern_file_lowers_exact_predicate_body() {
let patterns = parse_sem_patterns(
r#"
[[pattern]]
match = "isTypeName()"
lower = "bool(false)"
"#,
)
.expect("pattern file parses");
let predicates = structural_predicate_templates(
&predicate_parser_data(),
SemanticsKind::ParserPredicate,
&patterns,
)
.expect("pattern should lower predicate");
assert_eq!(predicates[0].1, PredicateTemplate::False);
}
#[test]
fn strip_toml_comment_respects_quoted_strings() {
// A `#` outside quotes starts a comment.
assert_eq!(strip_toml_comment("key = 1 # trailing"), "key = 1 ");
assert_eq!(strip_toml_comment("# whole line"), "");
// A `#` inside a basic or literal string is NOT a comment.
assert_eq!(
strip_toml_comment(r#"match = "text == '#'""#),
r#"match = "text == '#'""#
);
assert_eq!(
strip_toml_comment(r##"lower = 'str("#")'"##),
r##"lower = 'str("#")'"##
);
// A `#` after a closed string is still a comment.
assert_eq!(
strip_toml_comment(r#"match = "a" # note"#),
r#"match = "a" "#
);
// A `\"` escape inside a basic string does not close it, so a later `#`
// stays inside the string.
assert_eq!(
strip_toml_comment(r##"match = "a\"#b""##),
r##"match = "a\"#b""##
);
}
#[test]
fn sem_patterns_keep_hash_inside_quoted_match() {
// A `#` inside the quoted `match` body must survive parsing, not be
// truncated as a comment (which would silently change the pattern).
let patterns = parse_sem_patterns(
"[[pattern]]\nmatch = \"col == '#'\" # a real comment\nlower = \"bool(false)\"\n",
)
.expect("pattern file parses");
let template = patterns
.predicate_template(SemanticsKind::ParserPredicate, "col == '#'")
.expect("pattern lookup should not fail")
.expect("the '#'-bearing match body must be retained");
assert_eq!(template, PredicateTemplate::False);
}
#[test]
fn parse_toml_scalar_strips_single_quoted_literals() {
// TOML literal strings are single-quoted with no escape processing.
// `strip_toml_comment` already treats `'...'` as a string, so the scalar
// parser must strip those quotes too (verbatim body).
assert_eq!(parse_toml_scalar("'assume-false'"), "assume-false");
assert_eq!(parse_toml_scalar(" 'hook' "), "hook");
// A literal string is verbatim: a backslash is not an escape.
assert_eq!(parse_toml_scalar(r"'a\nb'"), r"a\nb");
// Double-quoted basic strings still unescape.
assert_eq!(parse_toml_scalar(r#""a\nb""#), "a\nb");
// A bare scalar and a lone quote are unchanged.
assert_eq!(parse_toml_scalar("42"), "42");
assert_eq!(parse_toml_scalar("'"), "'");
}
#[test]
fn sem_patterns_accept_single_quoted_dispose() {
// A single-quoted (TOML literal) `dispose` must resolve to the disposition,
// not keep its quotes (which would fail to match / be rejected).
let patterns = parse_sem_patterns(
"[[coordinate]]\nkind = 'predicate'\nrule = 's'\nindex = 0\ndispose = 'assume-false'\n",
)
.expect("pattern file parses");
assert_eq!(
patterns.coordinate_predicate_template(
SemanticsKind::ParserPredicate,
Some("s"),
Some(0),
),
Some(Some(PredicateTemplate::False)),
"single-quoted dispose must resolve to assume-false"
);
}
#[test]
fn semantic_helper_calls_capture_kind_negation_and_literals() {
assert_eq!(
parse_semantic_helper_call(
r#"!this.matches("marker", true, -2)"#,
SemanticsKind::LexerPredicate,
),
Some(SemanticHelperCall {
name: "matches".to_owned(),
arguments: vec![
SemanticLiteral::String("marker".to_owned()),
SemanticLiteral::Bool(true),
SemanticLiteral::Integer(-2),
],
negated: true,
})
);
assert_eq!(
parse_semantic_helper_call("this.HandleAction();", SemanticsKind::LexerAction,),
Some(SemanticHelperCall {
name: "HandleAction".to_owned(),
arguments: Vec::new(),
negated: false,
})
);
assert!(
parse_semantic_helper_call("this.n(dynamicValue)", SemanticsKind::ParserPredicate,)
.is_none()
);
assert_eq!(
parse_semantic_helper_call(
r"this.n('line\n\'quoted\'')",
SemanticsKind::ParserPredicate,
)
.expect("single-quoted literal parses")
.arguments,
[SemanticLiteral::String("line\n'quoted'".to_owned())]
);
}
#[test]
fn semantic_helper_patterns_are_scoped_by_kind_and_signature() {
let patterns = parse_sem_patterns(
"version = 1\n[[helper]]\nkind = \"lexer-action\"\nname = \"handle\"\nreturns = \"unit\"\nlower = \"hook\"\n[[helper]]\nname = \"n\"\narguments = \"string\"\nreturns = \"bool\"\nlower = \"hook\"\n",
)
.expect("pattern file parses");
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerAction, "this.handle();")
.expect("matching cannot fail")
.is_some()
);
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerPredicate, "this.handle()")
.expect("matching cannot fail")
.is_none()
);
let call = patterns
.hook_helper_call(SemanticsKind::ParserPredicate, r#"this.n("value")"#)
.expect("matching cannot fail")
.expect("string argument matches");
assert_eq!(
call.arguments,
[SemanticLiteral::String("value".to_owned())]
);
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerPredicate, r#"this.n("value")"#)
.expect("matching cannot fail")
.is_some()
);
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerAction, r#"this.n("value");"#)
.expect("matching cannot fail")
.is_none()
);
}
#[test]
fn lexer_typed_hook_signatures_reject_normalized_conflicts() {
let mappings = [
LexerTypedHookMapping {
rule_index: 0,
coordinate_index: 0,
kind: LexerTypedHookKind::Action,
method_name: "handle_value".to_owned(),
call: SemanticHelperCall {
name: "HandleValue".to_owned(),
arguments: vec![SemanticLiteral::String("x".to_owned())],
negated: false,
},
},
LexerTypedHookMapping {
rule_index: 1,
coordinate_index: 0,
kind: LexerTypedHookKind::Action,
method_name: "handle_value".to_owned(),
call: SemanticHelperCall {
name: "handle_value".to_owned(),
arguments: vec![SemanticLiteral::Integer(1)],
negated: false,
},
},
];
let error = validate_lexer_typed_hook_signatures(&mappings)
.expect_err("conflicting normalized signatures must fail generation");
assert!(error.to_string().contains("conflicting literal signatures"));
}
#[test]
fn coordinate_override_routes_parser_predicate_to_typed_hook() {
let patterns = parse_sem_patterns(
r#"
[[coordinate]]
kind = "predicate"
rule = "s"
index = 0
dispose = "hook"
"#,
)
.expect("pattern file parses");
let entries = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&patterns,
)
.expect("collection should succeed");
assert_eq!(entries[0].disposition, SemanticsDisposition::Hooked);
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
patterns: Some(&patterns),
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(module.contains("pub trait SParserHooks"));
assert!(module.contains("fn is_type_name"));
assert!(module.contains("(0, 0) => Some(self.0.is_type_name(ctx))"));
assert!(module.contains("PExpr::Hook"));
// The typed action escape hatch must report handled actions: the trait's
// `custom_action` returns `bool` and the adapter propagates it (so a
// typed action hook satisfies a hook/error policy instead of being
// treated as unhandled and failing loud).
assert!(
module.contains("_action: antlr4_runtime::ParserAction) -> bool"),
"custom_action must return a handled-bool"
);
assert!(
module.contains("self.0.custom_action(ctx, action)")
&& !module.contains("self.0.custom_action(ctx, action);"),
"the adapter must return custom_action's result, not discard it"
);
}
#[test]
fn typed_hook_mapping_skips_lexer_rule_predicates() {
// A combined grammar owns lexer and parser predicates structurally;
// only the parser predicate belongs in the parser hook adapter.
let data = parser_fixture_data("mixed-parser-lexer-predicates/S.g4");
let mappings = parser_typed_hook_mappings(&data, &SemPatternFile::default())
.expect("typed hook mapping should succeed");
// Only the parser-rule helper (`isTypeName` on rule `s`, pred 0) maps;
// the lexer-rule `aheadIsDigit` helper is not wired to a parser hook.
assert_eq!(
mappings.len(),
1,
"only the parser-rule helper maps: {mappings:?}"
);
assert_eq!((mappings[0].rule_index, mappings[0].pred_index), (0, 0));
assert_eq!(mappings[0].method_name, "is_type_name");
}
#[test]
fn typed_hook_predicate_method_name_avoids_action_hook_collision() {
// A grammar helper that normalizes to the reserved action-hook method
// name must be disambiguated, or the generated trait would declare two
// `custom_action` methods (Rust has no arity overloading).
assert_eq!(
typed_hook_predicate_method_name("customAction"),
"custom_action_pred"
);
assert_eq!(
typed_hook_predicate_method_name("custom_action"),
"custom_action_pred"
);
// An unrelated helper keeps its normalized name.
assert_eq!(
typed_hook_predicate_method_name("isTypeName"),
"is_type_name"
);
}
#[test]
fn typed_hook_adapter_disambiguates_custom_action_helper() {
// End-to-end: a bare `customAction()` predicate helper must not collide
// with the fixed `custom_action` action-hook method on the trait.
let data = parser_fixture_data("custom-action-predicate/S.g4");
let mappings = parser_typed_hook_mappings(&data, &SemPatternFile::default())
.expect("typed hook mapping should succeed");
assert_eq!(mappings.len(), 1);
assert_eq!(mappings[0].method_name, "custom_action_pred");
let adapter = render_typed_hook_adapter("SParser", &mappings);
// The predicate method is the disambiguated name; the action hook keeps
// the reserved name — two distinct methods, so the trait compiles.
assert!(adapter.contains("fn custom_action_pred<L>"));
assert!(adapter.contains(
"fn custom_action<L>(&mut self, _ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, _action: antlr4_runtime::ParserAction) -> bool"
));
assert!(adapter.contains("Some(self.0.custom_action_pred(ctx))"));
}
#[test]
fn manifest_predicate_provenance_skips_lexer_rule_block() {
// A combined grammar's parser manifest must use the parser predicate's
// structural provenance, never a lexer predicate's body.
let data = parser_fixture_data("mixed-parser-lexer-predicates/S.g4");
let entries = collect_parser_semantics(
&data,
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let predicate = entries
.iter()
.find(|entry| entry.kind == SemanticsKind::ParserPredicate)
.expect("parser predicate coordinate present");
assert_eq!(predicate.rule_name.as_deref(), Some("s"));
assert_eq!(
predicate.body.as_deref(),
Some("isTypeName()"),
"provenance must be the parser predicate body, not the lexer-rule aheadIsDigit()"
);
}
#[test]
fn require_full_semantics_rejects_policy_fallbacks_but_allows_hooks() {
let fallback = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
assert!(enforce_require_full_semantics(true, &fallback).is_err());
let mut hooked = fallback;
hooked[0].disposition = SemanticsDisposition::Hooked;
enforce_require_full_semantics(true, &hooked).expect("hooked coordinates are complete");
}
#[test]
fn collect_parser_semantics_inventories_untranslated_predicates() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
insta::assert_debug_snapshot!("untranslated_parser_predicate_semantics", entries);
}
#[test]
fn collect_parser_semantics_marks_supported_predicates_translated() {
let entries = collect_parser_semantics(
&translated_predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].disposition, SemanticsDisposition::Translated);
assert_eq!(entries[0].template.as_deref(), Some("True"));
}
#[test]
fn collect_parser_semantics_marks_helper_hooked_predicates_hooked() {
let patterns = parse_sem_patterns(
"version = 1\n\n[[helper]]\nname = \"isTypeName\"\nreturns = \"bool\"\nlower = \"hook\"\n",
)
.expect("pattern file should parse");
let entries =
collect_parser_semantics(&predicate_parser_data(), SemUnknownPolicy::Error, &patterns)
.expect("collection should succeed");
assert_eq!(entries.len(), 1);
// A helper routed to the hook trait is accounted for, but it is NOT a
// translation: the manifest must say `hooked` so users know the
// coordinate needs a runtime hook implementation.
assert_eq!(entries[0].disposition, SemanticsDisposition::Hooked);
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("hooked coordinates satisfy strict mode");
}
#[test]
fn enforce_sem_unknown_error_lists_untranslated_coordinates() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let error = enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect_err("untranslated predicate must fail generation");
let message = error.to_string();
insta::assert_snapshot!("untranslated_parser_predicate_error", message);
}
#[test]
fn enforce_sem_unknown_error_accepts_fully_translated_grammars() {
let entries = collect_parser_semantics(
&translated_predicate_parser_data(),
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("collection should succeed");
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("fully translated grammar passes strict mode");
}
#[test]
fn authored_empty_parser_action_is_explicit_noop() {
let data = parser_fixture_data("empty-parser-action/T.g4");
let entries =
collect_parser_semantics(&data, SemUnknownPolicy::Error, &SemPatternFile::default())
.expect("collection should succeed");
let action = entries
.iter()
.find(|entry| entry.kind == SemanticsKind::ParserAction)
.expect("action entry is inventoried");
let state = action
.atn_state
.expect("action state is bound structurally");
assert_eq!(action.body.as_deref(), Some(""));
assert_eq!(action.disposition, SemanticsDisposition::Synthetic);
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("empty authored action is a strict-policy no-op");
let module = render_parser_with_options(
"TParser",
&data,
ParserRenderOptions {
require_generated_parser: true,
embedded: false,
sem_unknown: SemUnknownPolicy::Error,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("empty action should not block generated parser output");
assert!(module.contains(&format!("{state} => {{}}")));
}
#[test]
fn parser_action_attribution_ignores_non_embedded_member_syntax() {
let data = parser_fixture_data("member-and-parser-action/T.g4");
let entries = collect_parser_semantics(
&data,
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("Java-style members are irrelevant to action attribution");
let action = entries
.iter()
.find(|entry| entry.kind == SemanticsKind::ParserAction)
.expect("action entry is inventoried");
assert!(action.atn_state.is_some());
assert_eq!(action.body.as_deref(), Some("native();"));
}
#[test]
fn enforce_sem_unknown_error_exempts_synthetic_actions() {
// A `Synthetic` action (ANTLR-inserted, e.g. LR elimination) must NOT
// fail under the Error policy: there is no author intent to implement.
let synthetic = SemanticsEntry {
kind: SemanticsKind::ParserAction,
rule_index: Some(1),
rule_name: Some("declarator".to_owned()),
index: None,
atn_state: Some(9),
line: None,
column: None,
body: None,
disposition: SemanticsDisposition::Synthetic,
template: None,
};
enforce_sem_unknown(SemUnknownPolicy::Error, std::slice::from_ref(&synthetic))
.expect("a synthetic action is exempt from the error gate");
// ...but an authored untranslated action (Ignored) at the same shape must fail.
let authored = SemanticsEntry {
disposition: SemanticsDisposition::Ignored,
..synthetic
};
enforce_sem_unknown(SemUnknownPolicy::Error, std::slice::from_ref(&authored))
.expect_err("an authored untranslated action must fail loud under error");
}
#[test]
fn enforce_sem_unknown_is_lenient_under_default_policy() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
enforce_sem_unknown(SemUnknownPolicy::AssumeTrue, &entries)
.expect("assume-true keeps the historical lenient behavior");
}
#[test]
fn enforce_sem_unknown_fails_per_coordinate_error_under_default_policy() {
// A per-coordinate `dispose = "error"` override must fail codegen even
// when the global policy is the lenient default; otherwise the
// coordinate lowers to no SemIR entry and silently falls back to
// AssumeTrue at runtime.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"error\"\n",
)
.expect("pattern file parses");
let entries = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&patterns,
)
.expect("collection should succeed");
let error = enforce_sem_unknown(SemUnknownPolicy::AssumeTrue, &entries)
.expect_err("per-coordinate error override must fail even under assume-true");
assert!(
error.to_string().contains("pred_index=0"),
"message should name the rejected coordinate: {error}"
);
}
#[test]
fn semantics_manifest_renders_coordinates_and_policy() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let manifest = render_semantics_manifest(
SemUnknownPolicy::AssumeTrue,
&[],
&[("parser", "SParser".to_owned(), entries)],
);
insta::assert_snapshot!("semantics_manifest_with_untranslated_predicate", manifest);
}
#[test]
fn translates_portable_boolean_local_semantics() {
let data = portable_bool_parser_data();
let portable = build_structural_portable_local_data(&data, &SemPatternFile::default())
.expect("portable local semantics should build");
let action_state = structural_actions(&data)
.expect("structural actions should resolve")
.into_iter()
.next()
.expect("portable fixture has one action")
.state;
assert_eq!(
portable.declarations,
[vec!["let mut __antlr_local_seen = false;".to_owned()]]
);
assert_eq!(portable.required_generated_rules, BTreeSet::from([0]));
assert_eq!(
portable
.inline_actions
.get(&action_state)
.map(String::as_str),
Some("__antlr_local_seen = true;")
);
assert_eq!(
portable
.predicates
.get(&(0, 0))
.map(|(condition, message)| (condition.as_str(), message.as_deref())),
Some(("__antlr_local_seen", Some("not seen")))
);
let module = render_parser("SParser", &data).expect("portable grammar should render");
assert!(module.contains("let mut __antlr_local_seen = false;"));
assert!(module.contains("__antlr_local_seen = true;"));
assert!(module.contains("if !(__antlr_local_seen) {"));
assert!(module.contains("failed_predicate_option_error(0, \"not seen\".to_owned())"));
let entries =
collect_parser_semantics(&data, SemUnknownPolicy::Error, &SemPatternFile::default())
.expect("portable coordinates should be inventoried");
assert_eq!(entries.len(), 2);
assert!(
entries
.iter()
.all(|entry| entry.disposition == SemanticsDisposition::Translated)
);
assert!(
entries
.iter()
.all(|entry| entry.template.as_deref() == Some("PortableBooleanLocal"))
);
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("portable coordinates satisfy strict semantics");
}
#[test]
fn semantics_manifest_renders_empty_inventory() {
let manifest = render_semantics_manifest(
SemUnknownPolicy::AssumeTrue,
&[],
&[("parser", "SParser".to_owned(), Vec::new())],
);
assert!(manifest.contains("\"coordinates\": []"));
}
#[test]
fn assume_false_policy_reaches_generated_runtime_options() {
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::AssumeFalse,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(module.contains(
"unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::AssumeFalse"
));
}
#[test]
fn generated_top_level_entry_surfaces_unknown_semantic_error() {
// The public generated entry must surface Error-policy coordinates the
// generated-direct predicate path recorded, or a parse that consulted an
// unimplemented hook predicate returns a recovered Ok tree instead of
// AntlrError::Unsupported.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::AssumeFalse,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("parser should render");
let surface_at = module
.find("if let Some(error) = self.base.take_unknown_semantic_error()")
.expect("generated top-level entry must surface recorded unknown-semantic coordinates");
// Guarded by the public entry only, not the nested (from-generated) path.
assert!(module.contains("if allow_generated_fallback {"));
// The fail-loud check must run before the generated entry can return Ok.
let ok_at = module[surface_at..]
.find("Ok(__tree)")
.map(|offset| surface_at + offset)
.expect("generated entry returns the tree after semantic checks");
assert!(surface_at < ok_at);
}
#[test]
fn generated_rule_error_prefers_recorded_semantic_error() {
// When a generated-direct predicate consulted an unimplemented hook
// (returning None under the Error policy), the alternative fails and
// `parse_generated_rule` returns a generic `failed_predicate_error`. The
// top-level `Err` arm must first drain any recorded fail-loud coordinate
// and return that `AntlrError::Unsupported`, otherwise the documented
// fail-loud error is shadowed by the generic rule error. The check is
// gated on `allow_generated_fallback` so a nested child keeps its hits for
// the generated parent to surface at its own boundary.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::Hook,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("parser should render");
// Locate the generated-rule `Err` arm's generic return.
let generic_return_at = module
.find("return Err(error.into_error());")
.expect("generated-rule Err arm returns the generic rule error");
// The fail-loud drain must appear inside that arm, before the generic
// return, under the top-level gate.
let arm_start = module[..generic_return_at]
.rfind("Err(error) => {")
.expect("generic return lives in the Err arm");
let arm = &module[arm_start..generic_return_at];
assert!(
arm.contains("if allow_generated_fallback {")
&& arm.contains(
"if let Some(semantic_error) = self.base.take_unknown_semantic_error()"
),
"the Err arm must drain a recorded semantic error before the generic return"
);
}
#[test]
fn interpreted_fallback_action_miss_is_surfaced_at_public_entry() {
// A public entry that falls back to the interpreted ATN path runs the
// non-buffered `run_action` loop immediately (an untranslated action
// routed to `parser_action_hook` records an `unhandled_action_hit` under
// the Error policy). The top-level surfacing check that drains those hits
// is gated on the SAME `allow_generated_fallback` condition as the branch
// that runs the interpreted fallback, so the miss cannot escape as `Ok`.
// (Verified end-to-end: parsing an untranslated action through the
// interpreted path under `--sem-unknown=hook` with a declining hook
// returns `AntlrError::Unsupported("unhandled semantic action: ...")`.)
//
// Emitting a *separate* check inside `parse_interpreted_rule_precedence`
// would be both dead (the outer check already drains the hits) and
// unsafe: an early `return Err` there would bypass the caller's ordinary
// generated-rule cleanup and error reporting path.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::Hook,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("parser should render");
// The interpreted call site in the top-level entry and the surfacing check
// share the `allow_generated_fallback` gate, so the surfacing check follows
// the interpreted call and drains any action-hook miss (or predicate miss)
// the fallback recorded.
let interpreted_call_at = module
.find("self.parse_interpreted_rule_precedence(rule_index, precedence)?")
.expect("top-level entry runs the interpreted fallback under allow_generated_fallback");
let surface_at = module[interpreted_call_at..]
.find("if let Some(error) = self.base.take_unknown_semantic_error()")
.map(|offset| interpreted_call_at + offset)
.expect("the public entry must drain recorded semantic misses after the fallback");
// Between the interpreted fallback call and the surfacing check the entry
// must not return `Ok`, or an action-hook miss recorded by the immediate
// `run_action` loop would escape as a recovered success.
assert!(
!module[interpreted_call_at..surface_at].contains("Ok(__tree)"),
"the entry must not return Ok between the interpreted fallback and the surfacing check"
);
// Both are under the same gate: the branch that runs the fallback and the
// check that drains its misses share `if allow_generated_fallback {`.
assert!(
module[..interpreted_call_at].contains("} else {"),
"the interpreted fallback remains the non-generated fallback path"
);
}
#[test]
fn hook_predicate_does_not_escalate_default_policy() {
// A per-coordinate `dispose = "hook"` predicate under the default global
// policy must NOT flip the whole parser to Error — that would turn
// unrelated `assume-true` coordinates into fail-loud. The hook falls
// through to the configured (default) policy per coordinate; users opt
// into fail-loud with --sem-unknown=error.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::AssumeTrue,
patterns: Some(&patterns),
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(
!module.contains("UnknownSemanticPolicy::Error"),
"a hook predicate must not escalate the default policy to Error"
);
}
#[test]
fn non_default_policy_installs_on_generated_parser_constructor() {
// The generated-direct predicate path reads BaseParser's
// `unknown_predicate_policy`, which the interpreter options never set on
// that path. The constructor must install a non-default policy so a
// generated rule's hook predicate honors --sem-unknown instead of the
// AssumeTrue default.
for (policy, literal) in [
(
SemUnknownPolicy::AssumeFalse,
"antlr4_runtime::UnknownSemanticPolicy::AssumeFalse",
),
(
SemUnknownPolicy::Error,
"antlr4_runtime::UnknownSemanticPolicy::Error",
),
] {
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: policy,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(
module.contains(&format!("base.set_unknown_predicate_policy({literal});")),
"policy {policy:?} must be installed on the generated constructor"
);
}
// The default policy leaves the constructor untouched (no needless call).
let default_module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions::default(),
)
.expect("parser should render");
assert!(!default_module.contains("set_unknown_predicate_policy"));
}
#[test]
fn default_policy_emits_assume_true_options_field() {
let module = render_parser_with_options(
"SParser",
&translated_predicate_parser_data(),
ParserRenderOptions::default(),
)
.expect("parser should render");
assert!(module.contains(
"unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::AssumeTrue"
));
}
#[test]
fn non_default_policy_disables_adaptive_direct_gate() {
// A grammar with no predicates/actions would normally allow the
// adaptive-direct shortcut, but that path drops the emitted
// ParserRuntimeOptions (and thus the policy). The gate must be disabled
// so a non-default policy always reaches the options-carrying call.
let default_module = render_parser_with_options("TParser", &minimal_parser_data(), {
ParserRenderOptions::default()
})
.expect("parser should render under default policy");
assert!(
default_module.contains("&& true && std::env::var_os(\"ANTLR4_RUST_ADAPTIVE_DIRECT\")"),
"the predicate-free fixture must allow adaptive-direct by default, or this test proves nothing"
);
for policy in [SemUnknownPolicy::AssumeFalse, SemUnknownPolicy::Error] {
let module = render_parser_with_options(
"TParser",
&minimal_parser_data(),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: policy,
patterns: None,
..ParserRenderOptions::default()
},
)
.expect("parser should render under a non-default policy");
assert!(
module.contains("&& false && std::env::var_os(\"ANTLR4_RUST_ADAPTIVE_DIRECT\")"),
"policy {policy:?} must disable the adaptive-direct gate"
);
}
}
#[test]
fn lexer_assume_false_policy_renders_failing_predicate_hook() {
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
false,
SemUnknownPolicy::AssumeFalse,
&SemPatternFile::default(),
false,
)
.expect("lexer should render");
assert!(module.contains("next_token_compiled_with_hooks"));
assert!(module.contains("|_, _| false"));
}
#[test]
fn lexer_per_coordinate_hook_override_routes_to_owned_hooks() {
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"lexer-predicate\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
false,
SemUnknownPolicy::AssumeTrue,
&patterns,
false,
)
.expect("per-coordinate hooks should render generated lexer plumbing");
assert!(module.contains("next_token_compiled_with_semantic_dispatch"));
assert!(module.contains("=> { None }"));
}
#[test]
fn lexer_per_coordinate_assume_false_override_renders_failing_arm() {
// A per-coordinate `dispose = "assume-false"` on an uncovered lexer
// predicate must render an explicit failing `run_predicate` arm and take
// the hook-taking token path even under the default global policy, so
// the override recorded in the manifest actually removes the guarded
// alternative at runtime.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"lexer-predicate\"\nindex = 0\ndispose = \"assume-false\"\n",
)
.expect("pattern file parses");
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
false,
SemUnknownPolicy::AssumeTrue,
&patterns,
false,
)
.expect("lexer should render");
// The uncovered coordinate now has an explicit `false` predicate arm and
// the lexer takes the hook-carrying token path (not next_token_compiled).
assert!(
module.contains("=> { Some(false) }"),
"override renders a failing arm"
);
assert!(module.contains("run_predicate"));
assert!(!module.contains("next_token_compiled(&mut self.base, sink, atn(), lexer_dfa())"));
}
#[test]
fn coordinate_override_applies_to_structural_predicate() {
// A `--sem-patterns` coordinate override replaces the body-derived
// structural predicate and reaches generated SemIR.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"assume-false\"\n",
)
.expect("pattern file parses");
let templates = structural_predicate_templates(
&predicate_parser_data(),
SemanticsKind::ParserPredicate,
&patterns,
)
.expect("override synthesis should succeed");
assert_eq!(templates, [((0, 0), PredicateTemplate::False)]);
// The rendered parser carries the SemIR predicate for that coordinate.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
ParserRenderOptions {
patterns: Some(&patterns),
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(
module.contains("rule_index: 0, pred_index: 0"),
"override-derived predicate must reach parser_semantics()"
);
}
#[test]
fn lexer_hook_policy_routes_uncovered_predicates_to_owned_hooks() {
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
false,
SemUnknownPolicy::Hook,
&SemPatternFile::default(),
false,
)
.expect("hook policy should render generated hook plumbing");
assert!(module.contains("next_token_compiled_with_semantic_dispatch"));
assert!(module.contains("hooks: H"));
}
#[test]
fn hook_disposed_lexer_actions_require_semantic_dispatch() {
let coordinates = [(0, 0)];
let rule_names = ["A".to_owned()];
let no_actions = [];
assert!(lexer_actions_require_semantic_hooks(
&coordinates,
&rule_names,
&no_actions,
&SemPatternFile::default(),
SemUnknownPolicy::Hook,
));
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"lexer-action\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
assert!(lexer_actions_require_semantic_hooks(
&coordinates,
&rule_names,
&no_actions,
&patterns,
SemUnknownPolicy::AssumeTrue,
));
let translated = [((0, 0), ActionTemplate::LexerPopMode)];
assert!(!lexer_actions_require_semantic_hooks(
&coordinates,
&rule_names,
&translated,
&SemPatternFile::default(),
SemUnknownPolicy::Hook,
));
}
#[test]
fn lexer_hook_disposition_is_full_semantics() {
let entry = SemanticsEntry {
kind: SemanticsKind::LexerAction,
rule_index: Some(0),
rule_name: Some("A".to_owned()),
index: Some(0),
atn_state: None,
line: None,
column: None,
body: Some("this.handle();".to_owned()),
disposition: SemanticsDisposition::Hooked,
template: None,
};
enforce_require_full_semantics(true, &[entry])
.expect("hooked lexer actions are implemented by generated hook plumbing");
}
#[test]
fn lexer_default_policy_keeps_compiled_token_path() {
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
false,
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
false,
)
.expect("lexer should render");
assert!(module.contains("next_token_compiled(&mut self.base, sink, atn(), lexer_dfa())"));
assert!(module.contains("if H::ENABLES_LEXER_LIFECYCLE"));
assert!(module.contains("next_token_compiled_with_semantic_dispatch"));
assert!(module.contains("pub fn reset(&mut self)"));
assert!(module.contains("reset_with_semantic_hooks"));
assert!(module.contains("pub fn set_input_stream(&mut self, input: I)"));
assert!(module.contains("set_input_stream_with_semantic_hooks"));
assert!(module.contains("self.base.set_input_stream(input)"));
assert!(module.contains("pub fn clear_dfa(&self)"));
assert!(module.contains("self.base.clear_dfa()"));
assert!(module.contains("pub fn add_error_listener<T>(&mut self, listener: T)"));
assert!(module.contains(
"T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,"
));
assert!(module.contains("self.base.add_error_listener(listener)"));
assert!(module.contains("pub fn remove_error_listeners(&mut self)"));
assert!(module.contains("self.base.remove_error_listeners()"));
assert!(module.contains(
"fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result<TokenId, TokenStoreError>"
));
assert!(module.contains(
"fn source_text(&self) -> Option<std::rc::Rc<str>> { self.base.source_text() }"
));
assert!(module.contains(
"fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool"
));
assert!(module.contains("Recognizer::notify_error_listeners("));
assert!(!module.contains("CommonToken"));
assert!(!module.contains("TokenFactory"));
}
#[test]
fn option_hook_requires_an_exact_assignment() {
assert_eq!(
normalize_option_hook(" superClass = BaseLexer ")
.expect("valid hook assignment should normalize"),
"superClass=BaseLexer"
);
assert!(normalize_option_hook("superClass").is_err());
assert!(normalize_option_hook("=BaseLexer").is_err());
assert!(normalize_option_hook("superClass=").is_err());
}
#[test]
fn lexer_run_predicate_default_arm_follows_policy() {
// A mixed lexer (one covered coordinate plus an uncovered one that
// lands on the catch-all arm) must honor `--sem-unknown`: assume-false
// rejects the uncovered predicate instead of leaving it viable.
let predicates = [((0_usize, 0_usize), PredicateTemplate::True)];
let assume_true = render_lexer_predicate_method(&predicates, SemUnknownPolicy::AssumeTrue);
assert!(assume_true.contains("_ => Some(true),"));
assert!(!assume_true.contains("_ => Some(false),"));
let assume_false =
render_lexer_predicate_method(&predicates, SemUnknownPolicy::AssumeFalse);
assert!(assume_false.contains("_ => Some(false),"));
assert!(!assume_false.contains("_ => Some(true),"));
// `error`/`hook` keep the conservative assume-true lexer default that
// matches historical behavior (lexer fail-loud is a codegen-time error,
// not a runtime catch-all).
let error = render_lexer_predicate_method(&predicates, SemUnknownPolicy::Error);
assert!(error.contains("_ => Some(true),"));
}
}