use std::collections::{BTreeMap, BTreeSet, btree_map::Entry};
use std::env;
use std::fmt::Write as _;
use std::fs;
use std::io::{self, Write as _};
use std::ops::AddAssign;
use std::path::{Path, PathBuf};
use antlr4_runtime::atn::lexer_dfa::CompiledLexerDfa;
use antlr4_runtime::atn::parser_atn::{
ParserAtn, ParserAtnState, ParserTokenSetKind, ParserTransition, ParserTransitionData,
};
use antlr4_runtime::atn::serialized::{AtnDeserializer, SerializedAtn};
use antlr4_runtime::atn::{AtnStateKind, LexerAction, LexerTransition};
use antlr4_runtime::token::TOKEN_EOF;
#[path = "../bin_support/embedded.rs"]
mod embedded;
#[path = "../bin_support/rust_names.rs"]
mod rust_names;
#[path = "../bin_support/templates.rs"]
mod templates;
#[cfg(test)]
use rust_names::is_rust_keyword;
use rust_names::{
module_name, rust_function_name, rust_string, rust_type_name, sanitize_identifier,
split_identifier_words,
};
use templates::{
is_after_action, is_definitions_action, is_init_action, is_members_action, is_options_block,
matching_action_brace, matching_template_close, next_predicate_action_block,
parse_template_string, split_template_arguments, template_sequence_bodies,
};
const GENERATED_MODULE_HEADER: &str = concat!(
"// @generated by ",
env!("CARGO_PKG_NAME"),
" v",
env!("CARGO_PKG_VERSION"),
" - do not edit\n",
"// project: ",
env!("CARGO_PKG_REPOSITORY"),
"\n",
"#[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]\n",
"#[rustfmt::skip]\n",
"mod __antlr4_rust_generated {\n",
"\n",
);
const GENERATED_MODULE_FOOTER: &str = "\
}
#[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]
pub use self::__antlr4_rust_generated::*;
";
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut args_iter = env::args().skip(1);
while let Some(arg) = args_iter.next() {
match arg.as_str() {
"--lexer" | "--parser" | "--lexer-name" | "--parser-name" | "--grammar"
| "--out-dir" | "--sem-patterns" | "--actions" | "--sem-unknown" | "--option-hook" => {
let _ = args_iter.next();
}
"--help" | "-h" => {
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", usage())?;
return Ok(());
}
_ => {}
}
}
let args = Args::parse()?;
fs::create_dir_all(&args.out_dir)?;
let grammar_source = args
.grammar
.as_deref()
.map(fs::read_to_string)
.transpose()?;
let grammar_options = grammar_source
.as_deref()
.map(|source| collect_grammar_options(source, &args.option_hooks))
.transpose()?
.unwrap_or_default();
emit_grammar_option_warnings(&grammar_options)?;
enforce_require_full_options(args.require_full_semantics, &grammar_options)?;
let mut manifest_grammars: Vec<(&'static str, String, Vec<SemanticsEntry>)> = Vec::new();
if let Some(lexer) = args.lexer {
let data = InterpData::parse(&fs::read_to_string(&lexer)?)?;
let grammar_name = args
.lexer_name
.clone()
.unwrap_or_else(|| grammar_name_from_path(&lexer));
// Embedded mode compiles bodies verbatim; the template-recognition
// manifest walk does not apply to rendered Rust sources.
let manifest_source = if args.embedded_actions {
None
} else {
grammar_source.as_deref()
};
let entries = collect_lexer_semantics(
&data,
manifest_source,
args.allow_unsupported_lexer_actions,
args.sem_unknown,
&args.sem_patterns,
)?;
enforce_sem_unknown(args.sem_unknown, &entries)?;
enforce_require_full_semantics(args.require_full_semantics, &entries)?;
let module = render_lexer(
&grammar_name,
&data,
grammar_source.as_deref(),
args.allow_unsupported_lexer_actions,
args.sem_unknown,
&args.sem_patterns,
args.embedded_actions,
)?;
fs::write(
args.out_dir
.join(format!("{}.rs", module_name(&grammar_name))),
module,
)?;
manifest_grammars.push(("lexer", grammar_name, entries));
}
if let Some(parser) = args.parser {
let data = InterpData::parse(&fs::read_to_string(&parser)?)?;
let grammar_name = args
.parser_name
.clone()
.unwrap_or_else(|| grammar_name_from_path(&parser));
let manifest_source = if args.embedded_actions {
None
} else {
grammar_source.as_deref()
};
let entries =
collect_parser_semantics(&data, manifest_source, args.sem_unknown, &args.sem_patterns)?;
enforce_sem_unknown(args.sem_unknown, &entries)?;
enforce_require_full_semantics(args.require_full_semantics, &entries)?;
let module = render_parser_with_options(
&grammar_name,
&data,
grammar_source.as_deref(),
ParserRenderOptions {
require_generated_parser: args.require_generated_parser,
embedded: args.embedded_actions,
sem_unknown: args.sem_unknown,
patterns: Some(&args.sem_patterns),
},
)?;
fs::write(
args.out_dir
.join(format!("{}.rs", module_name(&grammar_name))),
module,
)?;
manifest_grammars.push(("parser", grammar_name, entries));
}
write_semantics_manifest(
&args.out_dir,
args.sem_unknown,
&grammar_options,
&manifest_grammars,
)?;
Ok(())
}
/// Disposition policy for semantic predicate/action coordinates that the
/// generator cannot translate into runtime metadata.
///
/// Mirrors the runtime's `UnknownSemanticPolicy`; the generator additionally
/// uses [`Self::Error`] to fail code generation before emitting a module whose
/// semantics would be unreliable.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum SemUnknownPolicy {
/// Unknown predicates pass unconditionally and unknown actions are
/// no-ops. Matches the historical metadata-only behavior; deprecated as a
/// default and slated to change to [`Self::Error`] in a future minor
/// release.
#[default]
AssumeTrue,
/// Unknown predicates fail unconditionally; unknown actions remain
/// no-ops.
AssumeFalse,
/// Unknown coordinates are intentionally delegated to runtime hooks.
Hook,
/// Fail code generation when any semantic coordinate has no Rust
/// implementation.
Error,
}
impl SemUnknownPolicy {
fn parse_flag(value: &str) -> Result<Self, String> {
match value {
"error" => Ok(Self::Error),
"hook" => Ok(Self::Hook),
"assume-true" => Ok(Self::AssumeTrue),
"assume-false" => Ok(Self::AssumeFalse),
other => Err(format!(
"--sem-unknown accepts error, hook, assume-true, or assume-false; got {other}\n\n{}",
usage()
)),
}
}
const fn manifest_name(self) -> &'static str {
match self {
Self::AssumeTrue => "assume-true",
Self::AssumeFalse => "assume-false",
Self::Hook => "hook",
Self::Error => "error",
}
}
/// Manifest disposition recorded for a predicate coordinate that has no
/// translated template. [`Self::Error`] aborts generation before a
/// manifest is written, so its mapping is only ever read by the
/// fail-loud report.
const fn unknown_predicate_disposition(self) -> SemanticsDisposition {
match self {
Self::AssumeTrue | Self::Error => SemanticsDisposition::AssumeTrue,
Self::AssumeFalse => SemanticsDisposition::AssumeFalse,
Self::Hook => SemanticsDisposition::Hooked,
}
}
const fn unknown_action_disposition(self) -> SemanticsDisposition {
match self {
Self::Hook => SemanticsDisposition::Hooked,
Self::AssumeTrue | Self::AssumeFalse | Self::Error => SemanticsDisposition::Ignored,
}
}
}
/// Coordinate kinds tracked by the `semantics.json` manifest.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SemanticsKind {
LexerAction,
LexerPredicate,
ParserPredicate,
ParserAction,
}
impl SemanticsKind {
const fn manifest_name(self) -> &'static str {
match self {
Self::LexerAction => "lexer-action",
Self::LexerPredicate => "lexer-predicate",
Self::ParserPredicate => "parser-predicate",
Self::ParserAction => "parser-action",
}
}
const fn error_label(self) -> &'static str {
match self {
Self::LexerAction => "unsupported grammar action",
Self::LexerPredicate | Self::ParserPredicate => "unsupported semantic predicate",
Self::ParserAction => "unsupported grammar action",
}
}
}
/// How generation disposed of one semantic coordinate.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SemanticsDisposition {
/// A supported template translated the coordinate into runtime metadata
/// or generated Rust code.
Translated,
/// No implementation exists; recognition treats the predicate as passing.
AssumeTrue,
/// No implementation exists; recognition treats the predicate as failing.
AssumeFalse,
/// No generated implementation exists; runtime hooks own the coordinate.
Hooked,
/// The coordinate is intentionally rejected by policy.
Error,
/// No implementation exists; the action is a no-op at recognition time.
Ignored,
/// An action ANTLR synthesized (e.g. during left-recursion elimination),
/// not written by the grammar author. It is a no-op at recognition time
/// like [`Self::Ignored`], but carries no author intent, so it is exempt
/// from the `--sem-unknown=error` gate.
Synthetic,
}
impl SemanticsDisposition {
const fn manifest_name(self) -> &'static str {
match self {
Self::Translated => "translated",
Self::AssumeTrue => "assume-true",
Self::AssumeFalse => "assume-false",
Self::Hooked => "hooked",
Self::Error => "error",
Self::Ignored => "ignored",
Self::Synthetic => "synthetic",
}
}
}
/// How the Rust backend treats one top-level ANTLR grammar option.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GrammarOptionDisposition {
/// The ANTLR tool encoded the option's behavior into `.interp` metadata.
Metadata,
/// The option affects the upstream tool invocation, not Rust runtime
/// behavior.
ToolHandled,
/// Caller-owned Rust hooks explicitly provide the target behavior.
Hooked,
/// The option is legal ANTLR syntax but its target behavior is not
/// automatically implemented by the Rust backend.
Unsupported,
}
impl GrammarOptionDisposition {
const fn manifest_name(self) -> &'static str {
match self {
Self::Metadata => "metadata",
Self::ToolHandled => "tool-handled",
Self::Hooked => "hooked",
Self::Unsupported => "unsupported",
}
}
}
/// One source-level grammar option inventoried from `--grammar`.
#[derive(Clone, Debug, Eq, PartialEq)]
struct GrammarOptionEntry {
key: String,
value: String,
line: usize,
column: usize,
disposition: GrammarOptionDisposition,
}
impl GrammarOptionEntry {
fn assignment(&self) -> String {
format!("{}={}", self.key, self.value)
}
fn describe_unsupported(&self) -> String {
format!(
"unsupported grammar option: {} at {}:{} is accepted by ANTLR but is not \
automatically implemented by antlr4-rust-gen; target-specific behavior may be \
missing (see #98)",
self.assignment(),
self.line,
self.column
)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum CoordinateDispose {
Hook,
AssumeTrue,
AssumeFalse,
Error,
}
impl CoordinateDispose {
fn parse(value: &str) -> io::Result<Self> {
match value {
"hook" => Ok(Self::Hook),
"assume-true" => Ok(Self::AssumeTrue),
"assume-false" => Ok(Self::AssumeFalse),
"error" => Ok(Self::Error),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown coordinate dispose {other}"),
)),
}
}
const fn disposition(self) -> SemanticsDisposition {
match self {
Self::Hook => SemanticsDisposition::Hooked,
Self::AssumeTrue => SemanticsDisposition::AssumeTrue,
Self::AssumeFalse => SemanticsDisposition::AssumeFalse,
Self::Error => SemanticsDisposition::Error,
}
}
const fn predicate_template(self) -> Option<PredicateTemplate> {
match self {
Self::Hook => Some(PredicateTemplate::Hook),
Self::AssumeTrue => Some(PredicateTemplate::True),
Self::AssumeFalse => Some(PredicateTemplate::False),
Self::Error => None,
}
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
struct SemPatternFile {
patterns: Vec<SemPatternRule>,
helpers: Vec<SemHelperRule>,
coordinates: Vec<SemCoordinateOverride>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemPatternRule {
id: String,
match_body: String,
lower: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemHelperRule {
/// Explicit helper kind. A missing kind preserves the pre-v1 wildcard
/// behavior for lexer and parser predicates.
kind: Option<SemanticsKind>,
name: String,
arguments: Vec<SemanticLiteralKind>,
lower: String,
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
enum SemanticLiteralKind {
String,
Bool,
Integer,
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum SemanticLiteral {
String(String),
Bool(bool),
Integer(i64),
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemanticHelperCall {
name: String,
arguments: Vec<SemanticLiteral>,
negated: bool,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SemCoordinateOverride {
kind: SemanticsKind,
rule: Option<String>,
index: Option<usize>,
atn_state: Option<usize>,
dispose: CoordinateDispose,
}
impl SemPatternFile {
fn predicate_template(
&self,
kind: SemanticsKind,
body: &str,
) -> io::Result<Option<PredicateTemplate>> {
let body = body.trim();
let mut matches = Vec::new();
matches.extend(
self.patterns
.iter()
.filter(|pattern| pattern.match_body.trim() == body)
.map(|pattern| (pattern.id.as_str(), pattern.lower.as_str())),
);
matches.extend(
self.helpers
.iter()
.filter(|helper| semantic_helper_kind_matches(helper, kind))
.filter_map(|helper| {
parse_semantic_helper_call(body, kind)
.filter(|call| helper_call_matches(call, helper))
.map(|_| helper)
})
.map(|helper| (helper.name.as_str(), helper.lower.as_str())),
);
if matches.len() > 1 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"ambiguous semantic patterns for {body:?}: {}",
matches
.iter()
.map(|(id, _)| *id)
.collect::<Vec<_>>()
.join(", ")
),
));
}
matches
.first()
.map(|(_, lower)| parse_pattern_lower(lower))
.transpose()
}
fn hook_helper_call(
&self,
kind: SemanticsKind,
body: &str,
) -> io::Result<Option<SemanticHelperCall>> {
let Some(call) = parse_semantic_helper_call(body, kind) else {
return Ok(None);
};
let matches = self
.helpers
.iter()
.filter(|helper| {
semantic_helper_kind_matches(helper, kind) && helper.lower.trim() == "hook"
})
.filter(|helper| helper_call_matches(&call, helper))
.collect::<Vec<_>>();
if matches.len() > 1 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"ambiguous semantic helper patterns for {body:?}: {}",
matches
.iter()
.map(|helper| helper.name.as_str())
.collect::<Vec<_>>()
.join(", ")
),
));
}
Ok(matches.first().map(|_| call))
}
fn coordinate_disposition(
&self,
kind: SemanticsKind,
rule: Option<&str>,
index: Option<usize>,
atn_state: Option<usize>,
) -> Option<SemanticsDisposition> {
self.coordinate_override(kind, rule, index, atn_state)
.map(|override_| override_.dispose.disposition())
}
#[allow(clippy::option_option)]
fn coordinate_predicate_template(
&self,
kind: SemanticsKind,
rule: Option<&str>,
index: Option<usize>,
) -> Option<Option<PredicateTemplate>> {
self.coordinate_override(kind, rule, index, None)
.map(|override_| override_.dispose.predicate_template())
}
fn coordinate_override(
&self,
kind: SemanticsKind,
rule: Option<&str>,
index: Option<usize>,
atn_state: Option<usize>,
) -> Option<&SemCoordinateOverride> {
self.coordinates.iter().find(|override_| {
override_.kind == kind
&& override_
.rule
.as_deref()
.is_none_or(|expected| rule == Some(expected))
&& override_
.index
.is_none_or(|expected| index == Some(expected))
&& override_
.atn_state
.is_none_or(|expected| atn_state == Some(expected))
})
}
}
fn semantic_helper_kind_matches(helper: &SemHelperRule, kind: SemanticsKind) -> bool {
helper.kind == Some(kind)
|| (helper.kind.is_none()
&& matches!(
kind,
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate
))
}
fn helper_call_matches(call: &SemanticHelperCall, helper: &SemHelperRule) -> bool {
call.name == helper.name
&& call.arguments.len() == helper.arguments.len()
&& call
.arguments
.iter()
.zip(&helper.arguments)
.all(|(literal, kind)| {
matches!(
(literal, kind),
(SemanticLiteral::String(_), SemanticLiteralKind::String)
| (SemanticLiteral::Bool(_), SemanticLiteralKind::Bool)
| (SemanticLiteral::Integer(_), SemanticLiteralKind::Integer)
)
})
}
fn parse_semantic_helper_call(body: &str, kind: SemanticsKind) -> Option<SemanticHelperCall> {
let mut body = body.trim();
if matches!(
kind,
SemanticsKind::LexerAction | SemanticsKind::ParserAction
) {
body = body.strip_suffix(';').unwrap_or(body).trim_end();
}
let negated = matches!(
kind,
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate
) && body.starts_with('!');
if negated {
body = body[1..].trim_start();
}
body = body
.strip_prefix("this.")
.or_else(|| body.strip_prefix("self."))
.unwrap_or(body);
let open = body.find('(')?;
let name = body[..open].trim();
if !is_semantic_helper_identifier(name) || !body.ends_with(')') {
return None;
}
let arguments = parse_semantic_literals(&body[open + 1..body.len() - 1])?;
Some(SemanticHelperCall {
name: name.to_owned(),
arguments,
negated,
})
}
fn is_semantic_helper_identifier(value: &str) -> bool {
let mut chars = value.chars();
chars.next().is_some_and(|first| {
(first == '_' || first.is_ascii_alphabetic())
&& chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
})
}
fn parse_semantic_literals(body: &str) -> Option<Vec<SemanticLiteral>> {
let mut body = body.trim();
if body.is_empty() {
return Some(Vec::new());
}
let mut literals = Vec::new();
while !body.is_empty() {
if body.starts_with('"') || body.starts_with('\'') {
let quote = *body.as_bytes().first()?;
let mut escaped = false;
let mut end = None;
for (index, byte) in body.as_bytes().iter().copied().enumerate().skip(1) {
if escaped {
escaped = false;
} else if byte == b'\\' {
escaped = true;
} else if byte == quote {
end = Some(index);
break;
}
}
let end = end?;
let raw = &body[1..end];
let value = unescape_semantic_string(raw)?;
literals.push(SemanticLiteral::String(value));
body = body[end + 1..].trim_start();
} else {
let end = body.find(',').unwrap_or(body.len());
let raw = body[..end].trim();
let literal = match raw {
"true" => SemanticLiteral::Bool(true),
"false" => SemanticLiteral::Bool(false),
_ => SemanticLiteral::Integer(raw.parse().ok()?),
};
literals.push(literal);
body = body[end..].trim_start();
}
if body.is_empty() {
break;
}
body = body.strip_prefix(',')?.trim_start();
if body.is_empty() {
return None;
}
}
Some(literals)
}
fn unescape_semantic_string(value: &str) -> Option<String> {
let mut out = String::with_capacity(value.len());
let mut chars = value.chars();
while let Some(ch) = chars.next() {
if ch != '\\' {
out.push(ch);
continue;
}
match chars.next()? {
'"' => out.push('"'),
'\'' => out.push('\''),
'\\' => out.push('\\'),
'n' => out.push('\n'),
'r' => out.push('\r'),
't' => out.push('\t'),
other => {
out.push('\\');
out.push(other);
}
}
}
Some(out)
}
fn parse_pattern_lower(lower: &str) -> io::Result<PredicateTemplate> {
let lower = lower.trim();
match lower {
"true" | "bool(true)" => return Ok(PredicateTemplate::True),
"false" | "bool(false)" => return Ok(PredicateTemplate::False),
"hook" => return Ok(PredicateTemplate::Hook),
_ => {}
}
parse_pattern_lt_text(lower)
.or_else(|| parse_pattern_la_not_equals(lower))
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported semantic pattern lower expression {lower:?}"),
)
})
}
fn parse_pattern_lt_text(lower: &str) -> Option<PredicateTemplate> {
let body = lower.strip_prefix("cmp(eq, token_text(")?;
let (offset, rest) = body.split_once("), str(\"")?;
let text = rest.strip_suffix("\"))")?;
Some(PredicateTemplate::LookaheadTextEquals {
offset: offset.trim().parse().ok()?,
text: text.to_owned(),
})
}
fn parse_pattern_la_not_equals(lower: &str) -> Option<PredicateTemplate> {
let body = lower.strip_prefix("cmp(ne, la(")?;
let (offset, rest) = body.split_once("), token(")?;
let token_name = rest.strip_suffix("))")?;
Some(PredicateTemplate::LookaheadNotEquals {
offset: offset.trim().parse().ok()?,
token_name: token_name.trim().to_owned(),
})
}
/// One semantic predicate/action coordinate inventoried for the manifest.
///
/// Every field except `kind` and `disposition` is best-effort: source spans
/// and bodies require `--grammar`, and parser actions are keyed by ATN state
/// because their source pairing is heuristic.
#[derive(Clone, Debug)]
struct SemanticsEntry {
kind: SemanticsKind,
rule_index: Option<usize>,
rule_name: Option<String>,
index: Option<usize>,
atn_state: Option<usize>,
line: Option<usize>,
column: Option<usize>,
body: Option<String>,
disposition: SemanticsDisposition,
template: Option<String>,
}
impl SemanticsEntry {
/// Renders the fail-loud error line for this coordinate, matching the
/// shape documented in the compatibility-boundary docs.
fn describe_unsupported(&self) -> String {
let mut message = String::from(self.kind.error_label());
message.push(':');
match (&self.rule_name, self.rule_index) {
(Some(name), Some(rule_index)) => {
let _ = write!(message, " rule={name}({rule_index})");
}
(None, Some(rule_index)) => {
let _ = write!(message, " rule_index={rule_index}");
}
_ => {}
}
if let Some(index) = self.index {
let label = match self.kind {
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate => "pred_index",
SemanticsKind::LexerAction | SemanticsKind::ParserAction => "action_index",
};
let _ = write!(message, " {label}={index}");
}
if let Some(atn_state) = self.atn_state {
let _ = write!(message, " atn_state={atn_state}");
}
if let (Some(line), Some(column)) = (self.line, self.column) {
let _ = write!(message, " at {line}:{column}");
}
if let Some(body) = &self.body {
let _ = write!(message, ": {{{body}}}");
}
message
}
}
/// Fails generation when coordinates must be rejected at codegen: either the
/// global `--sem-unknown=error` policy is active (every unimplemented
/// coordinate), or a per-coordinate `dispose = "error"` override rejects a
/// specific coordinate regardless of the global policy.
///
/// A per-coordinate error override otherwise lowers to no `SemIR` entry and
/// does not escalate the runtime policy, so without this it would silently fall
/// back to the global default (e.g. `AssumeTrue`) instead of failing.
fn enforce_sem_unknown(policy: SemUnknownPolicy, entries: &[SemanticsEntry]) -> io::Result<()> {
let unsupported = entries
.iter()
.filter(|entry| {
// A per-coordinate `dispose = "error"` override is always fatal.
if entry.disposition == SemanticsDisposition::Error {
return true;
}
// Under the global Error policy, any unimplemented coordinate is
// fatal — except a `Synthetic` action ANTLR inserted itself (no
// author intent to implement).
policy == SemUnknownPolicy::Error
&& !matches!(
entry.disposition,
SemanticsDisposition::Translated
| SemanticsDisposition::Hooked
| SemanticsDisposition::Synthetic
)
})
.collect::<Vec<_>>();
if unsupported.is_empty() {
return Ok(());
}
let mut message = String::new();
for entry in &unsupported {
message.push_str(&entry.describe_unsupported());
message.push('\n');
}
let _ = write!(
message,
"--sem-unknown=error: {} semantic coordinate(s) have no Rust implementation and are \
configured to fail; pass --grammar so supported templates can be translated, adjust a \
coordinate's `dispose`, or accept a documented fallback with \
--sem-unknown=assume-true / assume-false",
unsupported.len()
);
Err(io::Error::new(io::ErrorKind::InvalidData, message))
}
fn enforce_require_full_semantics(require: bool, entries: &[SemanticsEntry]) -> io::Result<()> {
if !require {
return Ok(());
}
let fallback = entries
.iter()
.filter(|entry| {
// A `Synthetic` action is an ANTLR internal, not a missing author
// semantic, so it does not count as an unimplemented fallback.
!matches!(
entry.disposition,
SemanticsDisposition::Translated
| SemanticsDisposition::Hooked
| SemanticsDisposition::Synthetic
)
})
.collect::<Vec<_>>();
if fallback.is_empty() {
return Ok(());
}
let mut message = String::new();
for entry in &fallback {
message.push_str(&entry.describe_unsupported());
message.push('\n');
}
let _ = write!(
message,
"--require-full-semantics: {} semantic coordinate(s) use policy fallback dispositions",
fallback.len()
);
Err(io::Error::new(io::ErrorKind::InvalidData, message))
}
fn emit_grammar_option_warnings(entries: &[GrammarOptionEntry]) -> io::Result<()> {
let mut stderr = io::stderr().lock();
for entry in entries
.iter()
.filter(|entry| entry.disposition == GrammarOptionDisposition::Unsupported)
{
writeln!(stderr, "warning: {}", entry.describe_unsupported())?;
}
Ok(())
}
fn enforce_require_full_options(require: bool, entries: &[GrammarOptionEntry]) -> io::Result<()> {
if !require {
return Ok(());
}
let unsupported = entries
.iter()
.filter(|entry| entry.disposition == GrammarOptionDisposition::Unsupported)
.collect::<Vec<_>>();
if unsupported.is_empty() {
return Ok(());
}
let mut message = String::new();
for entry in &unsupported {
message.push_str(&entry.describe_unsupported());
message.push('\n');
}
let _ = write!(
message,
"--require-full-semantics: {} grammar option(s) require caller-owned target behavior; \
acknowledge implemented behavior with --option-hook KEY=VALUE",
unsupported.len()
);
Err(io::Error::new(io::ErrorKind::InvalidData, message))
}
fn normalize_option_hook(value: &str) -> Result<String, String> {
let Some((key, option_value)) = value.split_once('=') else {
return Err(format!(
"--option-hook requires KEY=VALUE; got {value:?}\n\n{}",
usage()
));
};
let key = key.trim();
let option_value = option_value.trim();
let mut key_chars = key.chars();
if !key_chars
.next()
.is_some_and(|ch| ch == '_' || ch.is_ascii_alphabetic())
|| !key_chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
|| option_value.is_empty()
{
return Err(format!(
"--option-hook requires KEY=VALUE; got {value:?}\n\n{}",
usage()
));
}
Ok(format!("{key}={option_value}"))
}
/// Inventories every custom-action and predicate coordinate in a lexer
/// `.interp`, mirroring the pairing rules `render_lexer` uses so manifest
/// dispositions match what the generated module will do.
/// Manifest disposition for a predicate coordinate given its covering
/// template: hook-routed coordinates report `hooked` (the user trait owns
/// them), any other template is a real translation, and uncovered
/// coordinates fall back to the unknown policy.
const fn predicate_template_disposition(
template: Option<&PredicateTemplate>,
policy: SemUnknownPolicy,
) -> SemanticsDisposition {
match template {
// Unwrap a `<fail=...>` wrapper: disposition follows the inner template
// (a wrapped hook is still `Hooked`, not `Translated`).
Some(PredicateTemplate::WithFailMessage { inner, .. }) => {
predicate_template_disposition(Some(inner), policy)
}
Some(PredicateTemplate::Hook) => SemanticsDisposition::Hooked,
Some(_) => SemanticsDisposition::Translated,
None => policy.unknown_predicate_disposition(),
}
}
fn collect_lexer_semantics(
data: &InterpData,
grammar_source: Option<&str>,
allow_unsupported_lexer_actions: bool,
policy: SemUnknownPolicy,
patterns: &SemPatternFile,
) -> io::Result<Vec<SemanticsEntry>> {
let mut entries = Vec::new();
let action_coordinates = lexer_custom_actions(data)?;
if !action_coordinates.is_empty() {
let templates = grammar_source
.map(|source| {
lexer_action_templates(data, source, allow_unsupported_lexer_actions, patterns)
})
.transpose()?;
let blocks = grammar_source
.map(|source| lexer_action_source_blocks(source, &data.rule_names))
.unwrap_or_default();
for (position, (rule_index, action_index)) in action_coordinates.iter().enumerate() {
let template = templates
.as_ref()
.and_then(|templates| templates.get(position))
.map(|(_, template)| template);
let translated = matches!(template, Some(ActionTemplate::LexerPopMode));
let block = blocks.get(position);
let rule_index = usize::try_from(*rule_index).ok();
entries.push(SemanticsEntry {
kind: SemanticsKind::LexerAction,
rule_index,
rule_name: rule_index.and_then(|rule| data.rule_names.get(rule).cloned()),
index: usize::try_from(*action_index).ok(),
atn_state: None,
line: block.map(|(line, _, _)| *line),
column: block.map(|(_, column, _)| *column),
body: block.map(|(_, _, body)| body.clone()),
disposition: patterns
.coordinate_disposition(
SemanticsKind::LexerAction,
rule_index.and_then(|rule| data.rule_names.get(rule).map(String::as_str)),
usize::try_from(*action_index).ok(),
None,
)
.unwrap_or_else(|| match template {
Some(ActionTemplate::Hook(_)) => SemanticsDisposition::Hooked,
Some(ActionTemplate::LexerPopMode) => SemanticsDisposition::Translated,
Some(ActionTemplate::UnsupportedLexerAction { .. }) | None => {
policy.unknown_action_disposition()
}
}),
template: template
.filter(|_| translated)
.map(|template| format!("{template:?}")),
});
}
}
let predicate_coordinates = lexer_predicate_transitions(data)?;
if !predicate_coordinates.is_empty() {
let templates = grammar_source
.map(|source| lexer_predicate_templates(data, source, patterns))
.transpose()?
.unwrap_or_default();
let blocks = grammar_source
.map(|source| predicate_source_blocks(source, &data.rule_names))
.unwrap_or_default();
for (position, (rule_index, pred_index)) in predicate_coordinates.iter().enumerate() {
let template = templates
.iter()
.find(|((rule, pred), _)| rule == rule_index && pred == pred_index)
.map(|(_, template)| template);
let block = blocks.get(position);
entries.push(SemanticsEntry {
kind: SemanticsKind::LexerPredicate,
rule_index: Some(*rule_index),
rule_name: data.rule_names.get(*rule_index).cloned(),
index: Some(*pred_index),
atn_state: None,
line: block.map(|(line, _, _)| *line),
column: block.map(|(_, column, _)| *column),
body: block.map(|(_, _, body)| body.clone()),
disposition: patterns
.coordinate_disposition(
SemanticsKind::LexerPredicate,
data.rule_names.get(*rule_index).map(String::as_str),
Some(*pred_index),
None,
)
.unwrap_or_else(|| predicate_template_disposition(template, policy)),
template: template.map(|template| format!("{template:?}")),
});
}
}
Ok(entries)
}
/// Inventories every predicate coordinate and action-transition state in a
/// parser `.interp`, mirroring `render_parser_with_options` pairing.
fn collect_parser_semantics(
data: &InterpData,
grammar_source: Option<&str>,
policy: SemUnknownPolicy,
patterns: &SemPatternFile,
) -> io::Result<Vec<SemanticsEntry>> {
let mut entries = Vec::new();
let portable_locals = grammar_source
.map(|source| build_portable_local_data(data, source, patterns))
.transpose()?
.unwrap_or_default();
let predicate_coordinates = parser_predicate_transitions(data)?;
if !predicate_coordinates.is_empty() {
let templates = grammar_source
.map(|source| parser_predicate_templates(data, source, patterns))
.transpose()?
.unwrap_or_default();
let blocks = grammar_source
.map(|source| predicate_source_blocks(source, &data.rule_names))
.unwrap_or_default();
for (position, coordinate) in predicate_coordinates.iter().enumerate() {
let template = templates
.iter()
.find(|(covered, _)| covered == coordinate)
.map(|(_, template)| template);
let block = blocks.get(position);
let (rule_index, pred_index) = *coordinate;
entries.push(SemanticsEntry {
kind: SemanticsKind::ParserPredicate,
rule_index: Some(rule_index),
rule_name: data.rule_names.get(rule_index).cloned(),
index: Some(pred_index),
atn_state: None,
line: block.map(|(line, _, _)| *line),
column: block.map(|(_, column, _)| *column),
body: block.map(|(_, _, body)| body.clone()),
disposition: patterns
.coordinate_disposition(
SemanticsKind::ParserPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
None,
)
.unwrap_or_else(|| {
if portable_locals.predicates.contains_key(coordinate) {
SemanticsDisposition::Translated
} else {
predicate_template_disposition(template, policy)
}
}),
template: portable_locals
.predicates
.contains_key(coordinate)
.then(|| "PortableBooleanLocal".to_owned())
.or_else(|| template.map(|template| format!("{template:?}"))),
});
}
}
let action_states = parser_action_states(data)?;
if !action_states.is_empty() {
let state_rules = parser_action_state_rules(data)?;
// Action states ANTLR synthesized (e.g. left-recursion elimination) as
// opposed to author-written `{...}` blocks. A synthetic untranslated
// action carries no author intent, so it is exempt from the
// `--sem-unknown=error` gate; an authored untranslated action is not.
let synthetic_states = synthetic_parser_action_states(data, grammar_source)?;
// Pair each authored action source-span with the same ATN action state,
// so span/body provenance in the manifest stays keyed by state rather
// than raw block position.
let block_spans = grammar_source
.map(|source| {
assign_states_to_parser_action_slots(
data,
source,
parser_action_source_block_slots(source, &data.rule_names),
)
})
.transpose()?
.unwrap_or_default();
let empty_action_states = block_spans
.iter()
.filter_map(|(state, slot)| slot.body.trim().is_empty().then_some(*state))
.collect::<BTreeSet<_>>();
for state in &action_states {
let rule_index = state_rules.get(state).copied();
let block = block_spans
.iter()
.find(|(covered, _)| covered == state)
.map(|(_, slot)| slot);
entries.push(SemanticsEntry {
kind: SemanticsKind::ParserAction,
rule_index,
rule_name: rule_index.and_then(|rule| data.rule_names.get(rule).cloned()),
index: None,
atn_state: Some(*state),
line: block.map(|slot| slot.line),
column: block.map(|slot| slot.column),
body: block.map(|slot| one_line_action_body(&slot.body)),
disposition: patterns
.coordinate_disposition(
SemanticsKind::ParserAction,
rule_index.and_then(|rule| data.rule_names.get(rule).map(String::as_str)),
None,
Some(*state),
)
.unwrap_or_else(|| {
if portable_locals.inline_actions.contains_key(state) {
SemanticsDisposition::Translated
} else if synthetic_states.contains(state)
|| empty_action_states.contains(state)
{
// ANTLR-synthesized actions and authored empty action
// bodies are no-op states exempt from the error gate.
SemanticsDisposition::Synthetic
} else {
policy.unknown_action_disposition()
}
}),
template: portable_locals
.inline_actions
.contains_key(state)
.then(|| "PortableBooleanLocal".to_owned()),
});
}
}
Ok(entries)
}
/// Collects `(line, column, body)` for every semantic-predicate block that
/// belongs to `rule_names`, in grammar source order (the order ANTLR assigns
/// predicate indexes).
///
/// The manifest pairs these spans positionally with the recognizer's predicate
/// transitions, so a combined grammar must skip the other rule set's predicates
/// (the same `predicate_block_included` filter the template scan uses) — else a
/// lexer predicate's line/body would be attached to a parser coordinate in
/// `semantics.json` and the `--sem-unknown=error` diagnostics.
fn predicate_source_blocks(source: &str, rule_names: &[String]) -> Vec<(usize, usize, String)> {
let mut blocks = Vec::new();
let mut offset = 0;
while let Some(block) = next_predicate_action_block(source, offset) {
offset = block.after_brace;
if !predicate_block_included(source, block.open_brace, rule_names) {
continue;
}
let (line, column) = line_column(source, block.open_brace);
blocks.push((line, column, one_line_action_body(block.body)));
}
blocks
}
/// Collects `(line, column, body)` for every lexer custom-action block using
/// the same filter chain as `extract_lexer_action_templates`, so positions
/// pair 1:1 with serialized custom-action coordinates.
fn lexer_action_source_blocks(source: &str, rule_names: &[String]) -> Vec<(usize, usize, String)> {
let mut blocks = Vec::new();
let mut offset = 0;
while let Some(block) = next_action_block(source, offset) {
offset = block.after_brace;
if !is_lexer_custom_action_block(source, &block, rule_names) {
continue;
}
let (line, column) = line_column(source, block.open_brace);
blocks.push((line, column, one_line_action_body(block.body)));
}
blocks
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ParserActionSourceSlot {
rule_index: usize,
source_offset: usize,
line: usize,
column: usize,
body: String,
}
/// Collects one source-span slot for each authored parser action block. Parser
/// action bodies are no longer recognized as `StringTemplate` markup by the
/// generator, but their rule/source positions still drive ATN attribution.
fn parser_action_source_block_slots(
source: &str,
rule_names: &[String],
) -> Vec<ParserActionSourceSlot> {
let mut slots = Vec::new();
let mut offset = 0;
while let Some(block) = next_action_block(source, offset) {
offset = block.after_brace;
if !is_parser_rule_body_action_block(source, &block, Some(rule_names)) {
continue;
}
let Some(header) = statement_rule_header(source, block.open_brace) else {
continue;
};
let Some(rule_index) = rule_names.iter().position(|name| name == header.name) else {
continue;
};
let (line, column) = line_column(source, block.open_brace);
slots.push(ParserActionSourceSlot {
rule_index,
source_offset: block.open_brace,
line,
column,
body: block.body.to_owned(),
});
}
slots
}
/// Converts a byte offset into a 1-based line and 0-based column, matching
/// ANTLR's source position convention.
fn line_column(source: &str, offset: usize) -> (usize, usize) {
let offset = offset.min(source.len());
let prefix = &source[..offset];
let line = prefix.bytes().filter(|byte| *byte == b'\n').count() + 1;
let column = prefix
.rfind('\n')
.map_or(offset, |newline| offset - newline - 1);
(line, column)
}
/// Inventories top-level `options { ... }` assignments from grammar source.
///
/// Rule-level options are excluded by requiring `options` to be the first
/// identifier after the previous grammar statement boundary.
fn collect_grammar_options(
source: &str,
option_hooks: &BTreeSet<String>,
) -> io::Result<Vec<GrammarOptionEntry>> {
let mut entries = Vec::new();
let mut offset = 0;
while let Some(open_brace) = templates::find_significant_open_brace(source, offset) {
let close_brace = matching_grammar_block(source, open_brace).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unterminated grammar block at byte {open_brace}"),
)
})?;
offset = close_brace + 1;
if !is_options_block(source, open_brace) {
continue;
}
let statement_start = statement_start_before(source, open_brace);
if rule_header_start_identifier(&source[statement_start..open_brace]) != Some("options") {
continue;
}
collect_grammar_option_block(
source,
open_brace + 1,
close_brace,
option_hooks,
&mut entries,
);
}
Ok(entries)
}
fn matching_grammar_block(source: &str, open_brace: usize) -> Option<usize> {
let mut cursor = templates::GrammarSourceCursor::new(source, open_brace + 1);
let mut nested = 0_usize;
while let Some((index, ch)) = cursor.next_significant() {
match ch {
'{' => nested += 1,
'}' if nested == 0 => return Some(index),
'}' => nested -= 1,
_ => {}
}
}
None
}
fn collect_grammar_option_block(
source: &str,
body_start: usize,
body_stop: usize,
option_hooks: &BTreeSet<String>,
entries: &mut Vec<GrammarOptionEntry>,
) {
let body = &source[body_start..body_stop];
let mut cursor = templates::GrammarSourceCursor::new(body, 0);
let mut assignment_start = 0;
while let Some((index, ch)) = cursor.next_significant() {
if ch == ';' {
collect_grammar_option_assignment(
source,
body_start,
&body[assignment_start..index],
assignment_start,
option_hooks,
entries,
);
assignment_start = index + 1;
}
}
collect_grammar_option_assignment(
source,
body_start,
&body[assignment_start..],
assignment_start,
option_hooks,
entries,
);
}
fn collect_grammar_option_assignment(
source: &str,
body_start: usize,
assignment: &str,
assignment_offset: usize,
option_hooks: &BTreeSet<String>,
entries: &mut Vec<GrammarOptionEntry>,
) {
let mut cursor = templates::GrammarSourceCursor::new(assignment, 0);
let mut first = None;
while let Some((index, ch)) = cursor.next_significant() {
if !ch.is_ascii_whitespace() {
first = Some((index, ch));
break;
}
}
let Some((key_start, first)) = first else {
return;
};
if first != '_' && !first.is_ascii_alphabetic() {
return;
}
let mut key_stop = key_start + first.len_utf8();
while assignment[key_stop..]
.chars()
.next()
.is_some_and(|ch| ch == '_' || ch.is_ascii_alphanumeric())
{
key_stop += assignment[key_stop..]
.chars()
.next()
.map_or(0, char::len_utf8);
}
cursor.seek(key_stop);
let mut equals = None;
while let Some((index, ch)) = cursor.next_significant() {
if ch == '=' {
equals = Some(index);
break;
}
}
let Some(equals) = equals else {
return;
};
let key = assignment[key_start..key_stop].to_owned();
let value = assignment[equals + 1..].trim().to_owned();
if value.is_empty() {
return;
}
let absolute_key = body_start + assignment_offset + key_start;
let (line, column) = line_column(source, absolute_key);
let assignment = format!("{key}={value}");
let disposition = match key.as_str() {
"tokenVocab" | "caseInsensitive" => GrammarOptionDisposition::Metadata,
"language" => GrammarOptionDisposition::ToolHandled,
_ if option_hooks.contains(&assignment) => GrammarOptionDisposition::Hooked,
_ => GrammarOptionDisposition::Unsupported,
};
entries.push(GrammarOptionEntry {
key,
value,
line,
column,
disposition,
});
}
/// Writes the `semantics.json` compatibility manifest next to the generated
/// modules. The manifest lists every semantic coordinate and how generation
/// disposed of it, making the supported/unsupported boundary inspectable.
fn write_semantics_manifest(
out_dir: &Path,
policy: SemUnknownPolicy,
options: &[GrammarOptionEntry],
grammars: &[(&'static str, String, Vec<SemanticsEntry>)],
) -> io::Result<()> {
fs::write(
out_dir.join("semantics.json"),
render_semantics_manifest(policy, options, grammars),
)
}
fn render_semantics_manifest(
policy: SemUnknownPolicy,
options: &[GrammarOptionEntry],
grammars: &[(&'static str, String, Vec<SemanticsEntry>)],
) -> String {
const DEPRECATION_NOTE: &str = "unknown coordinates currently default to assume-true; \
a future minor release changes the default to error";
let mut out = String::new();
out.push_str("{\n \"version\": 2,\n");
let _ = writeln!(
out,
" \"policy\": {},",
json_string(policy.manifest_name())
);
let _ = writeln!(out, " \"note\": {},", json_string(DEPRECATION_NOTE));
out.push_str(" \"options\": [");
for (position, option) in options.iter().enumerate() {
if position > 0 {
out.push(',');
}
out.push_str("\n ");
write_grammar_option_entry(&mut out, option);
}
if options.is_empty() {
out.push_str("],\n");
} else {
out.push_str("\n ],\n");
}
out.push_str(" \"grammars\": [");
for (grammar_position, (kind, name, entries)) in grammars.iter().enumerate() {
if grammar_position > 0 {
out.push(',');
}
out.push_str("\n {\n");
let _ = writeln!(out, " \"kind\": {},", json_string(kind));
let _ = writeln!(out, " \"name\": {},", json_string(name));
out.push_str(" \"coordinates\": [");
for (entry_position, entry) in entries.iter().enumerate() {
if entry_position > 0 {
out.push(',');
}
out.push_str("\n ");
write_semantics_entry(&mut out, entry);
}
if entries.is_empty() {
out.push_str("]\n }");
} else {
out.push_str("\n ]\n }");
}
}
if grammars.is_empty() {
out.push_str("]\n}\n");
} else {
out.push_str("\n ]\n}\n");
}
out
}
fn write_grammar_option_entry(out: &mut String, entry: &GrammarOptionEntry) {
out.push('{');
let _ = write!(out, "\"name\": {}", json_string(&entry.key));
let _ = write!(out, ", \"value\": {}", json_string(&entry.value));
let _ = write!(out, ", \"line\": {}", entry.line);
let _ = write!(out, ", \"column\": {}", entry.column);
let _ = write!(
out,
", \"disposition\": {}",
json_string(entry.disposition.manifest_name())
);
out.push('}');
}
fn write_semantics_entry(out: &mut String, entry: &SemanticsEntry) {
out.push('{');
let _ = write!(out, "\"kind\": {}", json_string(entry.kind.manifest_name()));
let _ = write!(
out,
", \"rule\": {}",
json_optional_string(entry.rule_name.as_deref())
);
let _ = write!(
out,
", \"rule_index\": {}",
json_optional_number(entry.rule_index)
);
let _ = write!(out, ", \"index\": {}", json_optional_number(entry.index));
let _ = write!(
out,
", \"atn_state\": {}",
json_optional_number(entry.atn_state)
);
let _ = write!(out, ", \"line\": {}", json_optional_number(entry.line));
let _ = write!(out, ", \"column\": {}", json_optional_number(entry.column));
let _ = write!(
out,
", \"body\": {}",
json_optional_string(entry.body.as_deref())
);
let _ = write!(
out,
", \"disposition\": {}",
json_string(entry.disposition.manifest_name())
);
let _ = write!(
out,
", \"template\": {}",
json_optional_string(entry.template.as_deref())
);
out.push('}');
}
fn json_optional_number(value: Option<usize>) -> String {
value.map_or_else(|| "null".to_owned(), |number| number.to_string())
}
fn json_optional_string(value: Option<&str>) -> String {
value.map_or_else(|| "null".to_owned(), json_string)
}
/// Escapes a string for JSON output; the generator hand-rolls this to avoid a
/// serialization dependency in a binary meant to be vendored into pipelines.
fn json_string(value: &str) -> String {
let mut out = String::with_capacity(value.len() + 2);
out.push('"');
for ch in value.chars() {
match ch {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
control if (control as u32) < 0x20 => {
let _ = write!(out, "\\u{:04x}", control as u32);
}
other => out.push(other),
}
}
out.push('"');
out
}
fn load_sem_patterns(path: &Path) -> io::Result<SemPatternFile> {
parse_sem_patterns(&fs::read_to_string(path)?)
}
/// Removes a trailing `#` comment from one TOML line, ignoring a `#` that
/// appears inside a quoted string (basic `"..."` or literal `'...'`). A naive
/// `split_once('#')` would corrupt valid pattern lines such as
/// `match = "text == '#'"` or a `str("#")` lower expression. Basic-string escape
/// handling means a `\"` inside `"..."` does not close the string.
fn strip_toml_comment(line: &str) -> &str {
let bytes = line.as_bytes();
let mut index = 0;
let mut quote: Option<u8> = None;
while index < bytes.len() {
let byte = bytes[index];
match quote {
// Inside a basic string, a backslash escapes the next byte.
Some(b'"') if byte == b'\\' => {
index += 2;
continue;
}
Some(q) if byte == q => quote = None,
Some(_) => {}
None if byte == b'"' || byte == b'\'' => quote = Some(byte),
None if byte == b'#' => return &line[..index],
None => {}
}
index += 1;
}
line
}
fn parse_sem_patterns(input: &str) -> io::Result<SemPatternFile> {
let mut file = SemPatternFile::default();
let mut section: Option<PatternSection> = None;
let mut fields = BTreeMap::<String, String>::new();
for raw_line in input.lines().chain(std::iter::once("")) {
let line = strip_toml_comment(raw_line).trim();
if line.is_empty() {
continue;
}
if let Some(next_section) = parse_pattern_section(line) {
flush_pattern_section(&mut file, section.take(), &mut fields)?;
section = Some(next_section);
continue;
}
let (key, value) = line.split_once('=').ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid semantic pattern line {line:?}"),
)
})?;
fields.insert(key.trim().to_owned(), parse_toml_scalar(value.trim()));
}
flush_pattern_section(&mut file, section, &mut fields)?;
Ok(file)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PatternSection {
Pattern,
Helper,
Coordinate,
}
fn parse_pattern_section(line: &str) -> Option<PatternSection> {
match line {
"[[pattern]]" => Some(PatternSection::Pattern),
"[[helper]]" => Some(PatternSection::Helper),
"[[coordinate]]" => Some(PatternSection::Coordinate),
_ => None,
}
}
fn flush_pattern_section(
file: &mut SemPatternFile,
section: Option<PatternSection>,
fields: &mut BTreeMap<String, String>,
) -> io::Result<()> {
let Some(section) = section else {
return Ok(());
};
match section {
PatternSection::Pattern => {
let match_body = take_required_field(fields, "match")?;
let lower = take_required_field(fields, "lower")?;
let id = fields
.remove("id")
.unwrap_or_else(|| format!("pattern:{}", file.patterns.len()));
file.patterns.push(SemPatternRule {
id,
match_body,
lower,
});
}
PatternSection::Helper => {
let kind = fields
.remove("kind")
.map(|value| parse_coordinate_kind(&value))
.transpose()?;
let arguments = fields
.remove("arguments")
.map_or_else(|| Ok(Vec::new()), |value| parse_helper_arguments(&value))?;
// `returns` is documentation in existing pattern files. Validate
// its shape when present, but the semantic kind determines whether
// the generated method returns bool or unit.
if let Some(returns) = fields.remove("returns") {
let expected = if kind.is_none_or(|kind| {
matches!(
kind,
SemanticsKind::LexerPredicate | SemanticsKind::ParserPredicate
)
}) {
"bool"
} else {
"unit"
};
if returns != expected {
let kind = kind.map_or("predicate", SemanticsKind::manifest_name);
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"semantic helper kind {kind} requires returns = {expected:?}, got {returns:?}"
),
));
}
}
file.helpers.push(SemHelperRule {
kind,
name: take_required_field(fields, "name")?,
arguments,
lower: take_required_field(fields, "lower")?,
});
}
PatternSection::Coordinate => {
file.coordinates.push(SemCoordinateOverride {
kind: parse_coordinate_kind(&take_required_field(fields, "kind")?)?,
rule: fields.remove("rule"),
index: fields
.remove("index")
.map(|value| parse_usize_field("index", &value))
.transpose()?,
atn_state: fields
.remove("atn_state")
.map(|value| parse_usize_field("atn_state", &value))
.transpose()?,
dispose: CoordinateDispose::parse(&take_required_field(fields, "dispose")?)?,
});
}
}
fields.clear();
Ok(())
}
fn parse_helper_arguments(value: &str) -> io::Result<Vec<SemanticLiteralKind>> {
let value = value.trim();
if value.is_empty() {
return Ok(Vec::new());
}
value
.split(',')
.map(|part| match part.trim() {
"string" | "str" => Ok(SemanticLiteralKind::String),
"bool" | "boolean" => Ok(SemanticLiteralKind::Bool),
"int" | "integer" => Ok(SemanticLiteralKind::Integer),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown semantic helper argument kind {other:?}"),
)),
})
.collect()
}
fn take_required_field(fields: &mut BTreeMap<String, String>, name: &str) -> io::Result<String> {
fields.remove(name).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("semantic pattern section missing {name}"),
)
})
}
fn parse_toml_scalar(value: &str) -> String {
let value = value.trim();
if let Some(body) = value
.strip_prefix('"')
.and_then(|body| body.strip_suffix('"'))
{
return unescape_toml_basic_string(body);
}
// TOML literal strings are single-quoted with no escape processing; the body
// is taken verbatim. `strip_toml_comment` already treats `'...'` as a quoted
// string, so a value like `dispose = 'assume-false'` must have its quotes
// stripped here too, or the quotes would leak into the disposition/match and
// it would be rejected or fail to match. Require length >= 2 so a lone `'`
// is not mistaken for an empty literal.
if value.len() >= 2 {
if let Some(body) = value
.strip_prefix('\'')
.and_then(|body| body.strip_suffix('\''))
{
return body.to_owned();
}
}
value.to_owned()
}
fn unescape_toml_basic_string(value: &str) -> String {
let mut out = String::with_capacity(value.len());
let mut escaped = false;
for ch in value.chars() {
if escaped {
match ch {
'"' => out.push('"'),
'\\' => out.push('\\'),
'n' => out.push('\n'),
'r' => out.push('\r'),
't' => out.push('\t'),
other => {
out.push('\\');
out.push(other);
}
}
escaped = false;
} else if ch == '\\' {
escaped = true;
} else {
out.push(ch);
}
}
if escaped {
out.push('\\');
}
out
}
fn parse_usize_field(name: &str, value: &str) -> io::Result<usize> {
value.parse::<usize>().map_err(|error| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid {name} value {value:?}: {error}"),
)
})
}
fn parse_coordinate_kind(value: &str) -> io::Result<SemanticsKind> {
match value {
"lexer-action" => Ok(SemanticsKind::LexerAction),
"lexer-predicate" => Ok(SemanticsKind::LexerPredicate),
"parser-predicate" | "predicate" => Ok(SemanticsKind::ParserPredicate),
"parser-action" | "action" => Ok(SemanticsKind::ParserAction),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown semantic coordinate kind {other}"),
)),
}
}
#[derive(Debug)]
struct Args {
lexer: Option<PathBuf>,
parser: Option<PathBuf>,
lexer_name: Option<String>,
parser_name: Option<String>,
grammar: Option<PathBuf>,
out_dir: PathBuf,
require_generated_parser: bool,
allow_unsupported_lexer_actions: bool,
sem_unknown: SemUnknownPolicy,
sem_patterns: SemPatternFile,
require_full_semantics: bool,
option_hooks: BTreeSet<String>,
/// `--actions embedded`: the grammar contains real Rust action/predicate
/// bodies (rendered through a `.test.stg`); splice them verbatim after
/// `$`-attribute translation instead of recognizing template markup.
embedded_actions: bool,
}
impl Args {
/// Parses the small generator CLI surface without pulling in a command-line
/// dependency.
///
/// This binary is intended to stay easy to vendor into build pipelines, so
/// the parser deliberately accepts only the flags the runtime target needs
/// today: lexer/parser `.interp` inputs, optional grammar names, and an
/// output directory.
fn parse() -> Result<Self, String> {
let mut lexer = None;
let mut parser = None;
let mut lexer_name = None;
let mut parser_name = None;
let mut grammar = None;
let mut out_dir = None;
let mut require_generated_parser = false;
let mut allow_unsupported_lexer_actions = false;
let mut embedded_actions = false;
let mut sem_unknown = SemUnknownPolicy::default();
let mut sem_patterns = SemPatternFile::default();
let mut require_full_semantics = false;
let mut option_hooks = BTreeSet::new();
let mut iter = env::args().skip(1);
while let Some(arg) = iter.next() {
match arg.as_str() {
"--lexer" => lexer = Some(PathBuf::from(next_arg(&mut iter, "--lexer")?)),
"--parser" => parser = Some(PathBuf::from(next_arg(&mut iter, "--parser")?)),
"--lexer-name" => lexer_name = Some(next_arg(&mut iter, "--lexer-name")?),
"--parser-name" => parser_name = Some(next_arg(&mut iter, "--parser-name")?),
"--grammar" => grammar = Some(PathBuf::from(next_arg(&mut iter, "--grammar")?)),
"--out-dir" => out_dir = Some(PathBuf::from(next_arg(&mut iter, "--out-dir")?)),
"--require-generated-parser" => require_generated_parser = true,
"--allow-unsupported-lexer-actions" => allow_unsupported_lexer_actions = true,
"--sem-patterns" => {
sem_patterns =
load_sem_patterns(&PathBuf::from(next_arg(&mut iter, "--sem-patterns")?))
.map_err(|error| format!("failed to load --sem-patterns: {error}"))?;
}
"--require-full-semantics" => require_full_semantics = true,
"--option-hook" => {
let value = next_arg(&mut iter, "--option-hook")?;
option_hooks.insert(normalize_option_hook(&value)?);
}
"--actions" => {
let value = next_arg(&mut iter, "--actions")?;
match value.as_str() {
"embedded" => embedded_actions = true,
"templates" => embedded_actions = false,
other => {
return Err(format!(
"unknown --actions mode {other:?} (expected embedded|templates)"
));
}
}
}
"--sem-unknown" => {
sem_unknown =
SemUnknownPolicy::parse_flag(&next_arg(&mut iter, "--sem-unknown")?)?;
}
"--help" | "-h" => return Err(usage()),
other => return Err(format!("unknown argument {other}\n\n{}", usage())),
}
}
if lexer.is_none() && parser.is_none() {
return Err(format!(
"at least one of --lexer or --parser is required\n\n{}",
usage()
));
}
Ok(Self {
lexer,
parser,
lexer_name,
parser_name,
grammar,
out_dir: out_dir.unwrap_or_else(|| PathBuf::from(".")),
require_generated_parser,
allow_unsupported_lexer_actions,
sem_unknown,
sem_patterns,
require_full_semantics,
option_hooks,
embedded_actions,
})
}
}
fn next_arg(iter: &mut impl Iterator<Item = String>, flag: &str) -> Result<String, String> {
iter.next()
.ok_or_else(|| format!("{flag} requires a value\n\n{}", usage()))
}
fn usage() -> String {
"\
Usage: antlr4-rust-gen [OPTIONS]
Options:
--lexer Lexer.interp Read lexer metadata from Lexer.interp
--parser Parser.interp Read parser metadata from Parser.interp
--lexer-name NAME Override the lexer grammar name
--parser-name NAME Override the parser grammar name
--grammar Grammar.g4 Read the source grammar for semantic metadata
--out-dir DIR Write generated Rust files to DIR
--actions embedded|templates Select real embedded actions or template metadata
--require-generated-parser Require parser source for parser generation
--allow-unsupported-lexer-actions
Ignore unsupported lexer actions
--sem-unknown error|hook|assume-true|assume-false
Choose unsupported semantic predicate policy
--sem-patterns FILE Load semantic helper patterns
--option-hook KEY=VALUE Acknowledge an option implemented by caller hooks
--require-full-semantics Fail if any semantic coordinate or option is unsupported
-h, --help Print this help"
.to_owned()
}
#[derive(Clone, Debug, Default)]
struct InterpData {
literal_names: Vec<Option<String>>,
symbolic_names: Vec<Option<String>>,
rule_names: Vec<String>,
channel_names: Vec<String>,
mode_names: Vec<String>,
atn: Vec<i32>,
}
impl InterpData {
/// Parses ANTLR `.interp` files emitted next to generated grammars.
///
/// The `.interp` format is line-oriented metadata followed by one serialized
/// ATN integer array. We use it as the clean-room bridge from the official
/// ANTLR tool to generated Rust metadata without reading or translating
/// another target's generated source.
fn parse(input: &str) -> Result<Self, io::Error> {
let mut data = Self::default();
let mut section = Section::None;
let mut atn_text = String::new();
for line in input.lines() {
let trimmed = line.trim();
section = match trimmed {
"token literal names:" => Section::LiteralNames,
"token symbolic names:" => Section::SymbolicNames,
"rule names:" => Section::RuleNames,
"channel names:" => Section::ChannelNames,
"mode names:" => Section::ModeNames,
"atn:" => Section::Atn,
_ => section,
};
if matches!(
trimmed,
"token literal names:"
| "token symbolic names:"
| "rule names:"
| "channel names:"
| "mode names:"
| "atn:"
) {
continue;
}
match section {
Section::None => {}
Section::LiteralNames => data.literal_names.push(parse_optional_name(trimmed)),
Section::SymbolicNames => data.symbolic_names.push(parse_optional_name(trimmed)),
Section::RuleNames => {
if !trimmed.is_empty() {
data.rule_names.push(trimmed.to_owned());
}
}
Section::ChannelNames => {
if !trimmed.is_empty() {
data.channel_names.push(trimmed.to_owned());
}
}
Section::ModeNames => {
if !trimmed.is_empty() {
data.mode_names.push(trimmed.to_owned());
}
}
Section::Atn => atn_text.push_str(trimmed),
}
}
data.atn = parse_atn_values(&atn_text)?;
Ok(data)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Section {
None,
LiteralNames,
SymbolicNames,
RuleNames,
ChannelNames,
ModeNames,
Atn,
}
fn parse_optional_name(value: &str) -> Option<String> {
match value {
"" | "null" => None,
other => Some(other.to_owned()),
}
}
/// Parses the bracketed serialized ATN integer array from an `.interp` file.
fn parse_atn_values(value: &str) -> Result<Vec<i32>, io::Error> {
let body = value.trim().trim_start_matches('[').trim_end_matches(']');
if body.is_empty() {
return Ok(Vec::new());
}
body.split(',')
.map(|part| {
part.trim().parse::<i32>().map_err(|error| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid ATN integer {:?}: {error}", part.trim()),
)
})
})
.collect()
}
/// Sets the rendered template for a lexer predicate coordinate, replacing any
/// existing (translated) entry so a per-coordinate override wins over a
/// built-in translation, or appending a new entry for an uncovered coordinate.
fn set_lexer_predicate_template(
predicates: &mut Vec<((usize, usize), PredicateTemplate)>,
coordinate: (usize, usize),
template: PredicateTemplate,
) {
if let Some(entry) = predicates.iter_mut().find(|(pred, _)| *pred == coordinate) {
entry.1 = template;
} else {
predicates.push((coordinate, template));
}
}
/// Renders a Rust lexer module that delegates token recognition to the shared
/// ATN interpreter.
///
/// The emitted lexer owns only generated metadata and a `BaseLexer`. Keeping
/// recognition in the runtime avoids emitting thousands of lines of
/// grammar-specific Rust control flow for the first target implementation.
#[allow(clippy::too_many_arguments)]
fn render_lexer(
grammar_name: &str,
data: &InterpData,
grammar_source: Option<&str>,
allow_unsupported_lexer_actions: bool,
sem_unknown: SemUnknownPolicy,
patterns: &SemPatternFile,
embedded: bool,
) -> io::Result<String> {
let type_name = rust_type_name(grammar_name);
let metadata = render_metadata(grammar_name, data);
let token_constants = render_token_constants(data);
// Embedded mode: lexer action/predicate bodies are verbatim Rust from the
// rendered grammar; translate the recognizer surface textually and skip
// the template machinery entirely.
let template_grammar_source = if embedded { None } else { grammar_source };
let embedded_lexer_actions = if embedded {
grammar_source.map_or_else(
|| Ok(Vec::new()),
|source| embedded_lexer_actions(data, source),
)?
} else {
Vec::new()
};
let embedded_lexer_predicates = if embedded {
grammar_source.map_or_else(
|| Ok(Vec::new()),
|source| embedded_lexer_predicates(data, source),
)?
} else {
Vec::new()
};
let mut actions = template_grammar_source.map_or_else(
|| Ok(Vec::new()),
|source| lexer_action_templates(data, source, allow_unsupported_lexer_actions, patterns),
)?;
// Any per-coordinate override replaces a translated action. Hook overrides
// fall through to the semantic hook object; assume-* overrides are no-ops.
actions.retain(|((rule_index, action_index), _)| {
let rule_name = usize::try_from(*rule_index)
.ok()
.and_then(|rule| data.rule_names.get(rule).map(String::as_str));
patterns
.coordinate_override(
SemanticsKind::LexerAction,
rule_name,
usize::try_from(*action_index).ok(),
None,
)
.is_none()
});
let mut predicates = template_grammar_source.map_or_else(
|| Ok(Vec::new()),
|source| lexer_predicate_templates(data, source, patterns),
)?;
// Apply per-coordinate overrides to every lexer predicate transition. Hook
// coordinates are represented explicitly so generated dispatch can fall
// through to the caller-owned semantic hook object.
for coordinate in lexer_predicate_transitions(data)? {
let (rule_index, pred_index) = coordinate;
let dispose = patterns
.coordinate_override(
SemanticsKind::LexerPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
None,
)
.map(|override_| override_.dispose);
let covered = predicates.iter().any(|(pred, _)| *pred == coordinate);
match dispose {
Some(CoordinateDispose::Hook) => {
set_lexer_predicate_template(&mut predicates, coordinate, PredicateTemplate::Hook);
}
Some(CoordinateDispose::Error) => {}
Some(CoordinateDispose::AssumeFalse) => {
set_lexer_predicate_template(&mut predicates, coordinate, PredicateTemplate::False);
}
Some(CoordinateDispose::AssumeTrue) => {
set_lexer_predicate_template(&mut predicates, coordinate, PredicateTemplate::True);
}
None => {
if !covered && sem_unknown == SemUnknownPolicy::Hook {
set_lexer_predicate_template(
&mut predicates,
coordinate,
PredicateTemplate::Hook,
);
}
}
}
}
// Predicate coordinates with no translated template normally fall back to
// always-true; `--sem-unknown=assume-false` flips that fallback, which
// requires the hook-taking token path even when no dispatch is generated.
let unknown_predicates_assume_false = sem_unknown == SemUnknownPolicy::AssumeFalse
&& predicates.is_empty()
&& !lexer_predicate_transitions(data)?.is_empty();
let lexer_dfa_data = compiled_lexer_dfa_words(data);
let lexer_typed_hook_mappings = if embedded {
Vec::new()
} else {
lexer_typed_hook_mappings(data, grammar_source, patterns, &actions)?
};
let typed_hook_adapter =
render_lexer_typed_hook_adapter(&type_name, &lexer_typed_hook_mappings);
let typed_lexer_constructor = if lexer_typed_hook_mappings.is_empty() {
String::new()
} else {
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
format!(
"impl<I, T> {type_name}<I, {adapter_name}<T>>\nwhere\n I: CharStream,\n T: {trait_name},\n{{\n pub fn with_typed_hooks(input: I, hooks: T) -> Self {{\n Self::with_hooks(input, {adapter_name}::new(hooks))\n }}\n}}\n"
)
};
let action_coordinates = lexer_custom_actions(data)?;
let has_hook_disposed_actions = lexer_actions_require_semantic_hooks(
&action_coordinates,
&data.rule_names,
&actions,
patterns,
sem_unknown,
);
let has_semantic_hooks = has_hook_disposed_actions
|| !lexer_typed_hook_mappings.is_empty()
|| actions
.iter()
.any(|(_, template)| matches!(template, ActionTemplate::Hook(_)))
|| predicates
.iter()
.any(|(_, template)| matches!(template, PredicateTemplate::Hook));
let has_action_dispatch = if embedded {
embedded_lexer_actions
.iter()
.any(|(_, statement)| !statement.trim().is_empty())
} else {
lexer_actions_need_dispatch(&actions)
};
let action_method = if embedded {
render_embedded_lexer_action_method(&embedded_lexer_actions)
} else {
render_lexer_action_method(&actions)
};
let predicate_method = if embedded {
render_embedded_lexer_predicate_method(&embedded_lexer_predicates)
} else {
render_lexer_predicate_method(&predicates, sem_unknown)
};
let has_predicate_dispatch = if embedded {
!embedded_lexer_predicates.is_empty()
} else {
!predicates.is_empty()
};
let lexer_unknown_policy = match sem_unknown {
SemUnknownPolicy::AssumeTrue => "antlr4_runtime::UnknownSemanticPolicy::AssumeTrue",
SemUnknownPolicy::AssumeFalse => "antlr4_runtime::UnknownSemanticPolicy::AssumeFalse",
SemUnknownPolicy::Hook | SemUnknownPolicy::Error => {
"antlr4_runtime::UnknownSemanticPolicy::Error"
}
};
let semantic_action = if has_action_dispatch {
"Self::run_action"
} else {
"|_, _| false"
};
let semantic_predicate = if has_predicate_dispatch {
"Self::run_predicate"
} else {
"|_, _| None"
};
let lifecycle_next_token_call = format!(
"antlr4_runtime::atn::lexer::next_token_compiled_with_semantic_dispatch(&mut self.base, sink, atn(), lexer_dfa(), &mut self.hooks, {semantic_action}, {semantic_predicate}, {lexer_unknown_policy}, |_, _, _| {{}})"
);
let next_token_call = if has_semantic_hooks {
lifecycle_next_token_call.clone()
} else if !has_action_dispatch && !has_predicate_dispatch && !unknown_predicates_assume_false {
"antlr4_runtime::atn::lexer::next_token_compiled(&mut self.base, sink, atn(), lexer_dfa())"
.to_owned()
} else {
let action = if has_action_dispatch {
"|base, action| { let _ = Self::run_action(base, action); }"
} else {
"|_, _| {}"
};
let predicate = if has_predicate_dispatch {
"|base, predicate| Self::run_predicate(base, predicate).unwrap_or(true)"
} else if unknown_predicates_assume_false {
"|_, _| false"
} else {
"|_, _| true"
};
format!(
"antlr4_runtime::atn::lexer::next_token_compiled_with_hooks(&mut self.base, sink, atn(), lexer_dfa(), {action}, {predicate}, |_, _, _| {{}})"
)
};
let next_token_call = if has_semantic_hooks {
next_token_call
} else {
format!(
"if H::ENABLES_LEXER_LIFECYCLE {{\n {lifecycle_next_token_call}\n }} else {{\n {next_token_call}\n }}"
)
};
let generated_header = GENERATED_MODULE_HEADER;
let generated_footer = GENERATED_MODULE_FOOTER;
Ok(format!(
r#"{generated_header}use antlr4_runtime::char_stream::CharStream;
use antlr4_runtime::recognizer::RecognizerData;
use antlr4_runtime::token::{{TokenId, TokenSink, TokenSource, TokenStoreError}};
use antlr4_runtime::atn::LexerAtn;
use antlr4_runtime::atn::lexer_dfa::CompiledLexerDfa;
use antlr4_runtime::atn::serialized::AtnDeserializer;
use antlr4_runtime::{{BaseLexer, GeneratedLexer, GrammarMetadata, Lexer, Recognizer}};
use std::sync::OnceLock;
{token_constants}
{metadata}
{typed_hook_adapter}
static ATN_CELL: OnceLock<LexerAtn> = OnceLock::new();
/// Deserializes and caches the grammar ATN for all lexer instances.
fn atn() -> &'static LexerAtn {{
ATN_CELL.get_or_init(|| {{
let serialized = metadata().serialized_atn();
AtnDeserializer::new(&serialized)
.deserialize()
.expect("generated lexer contains a valid ANTLR serialized ATN")
}})
}}
static LEXER_DFA_DATA: &[u32] = &[{lexer_dfa_data}];
static LEXER_DFA_CELL: OnceLock<CompiledLexerDfa> = OnceLock::new();
/// Ahead-of-time lexer DFA tables compiled by antlr4-rust-gen, embedded so
/// runtime startup only deserializes them. Rebuilt from the ATN instead when
/// the embedded stream comes from a different runtime version.
fn lexer_dfa() -> &'static CompiledLexerDfa {{
LEXER_DFA_CELL.get_or_init(|| {{
CompiledLexerDfa::from_serialized(LEXER_DFA_DATA)
.unwrap_or_else(|| CompiledLexerDfa::compile(atn()))
}})
}}
#[derive(Clone, Debug)]
pub struct {type_name}<I, H = antlr4_runtime::NoSemanticHooks>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
base: BaseLexer<I>,
hooks: H,
}}
impl<I> {type_name}<I, antlr4_runtime::NoSemanticHooks>
where
I: CharStream,
{{
pub fn new(input: I) -> Self {{
Self::with_hooks(input, antlr4_runtime::NoSemanticHooks)
}}
}}
impl<I, H> {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
pub fn with_hooks(input: I, hooks: H) -> Self {{
let grammar_metadata = metadata();
let data = RecognizerData::new(
grammar_metadata.grammar_file_name(),
grammar_metadata.vocabulary(),
)
.with_rule_names(grammar_metadata.rule_names().iter().copied())
.with_channel_names(grammar_metadata.channel_names().iter().copied())
.with_mode_names(grammar_metadata.mode_names().iter().copied());
Self {{ base: BaseLexer::new(input, data).with_shared_dfa(atn()), hooks }}
}}
pub fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
/// Adds a listener for lexer diagnostics.
pub fn add_error_listener<T>(&mut self, listener: T)
where
T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,
{{
self.base.add_error_listener(listener);
}}
/// Removes every lexer error listener, including the default console listener.
pub fn remove_error_listeners(&mut self) {{
self.base.remove_error_listeners();
}}
/// Routes every token through ATN interpretation instead of the compiled
/// lexer DFA, so the learned-DFA trace (`lexer_dfa_string`) observes each
/// match.
pub fn set_force_interpreted(&mut self, force_interpreted: bool) {{
self.base.set_force_interpreted(force_interpreted);
}}
/// Resets this lexer and any caller-owned lifecycle state for reuse.
pub fn reset(&mut self) {{
if H::ENABLES_LEXER_LIFECYCLE {{
antlr4_runtime::atn::lexer::reset_with_semantic_hooks(
&mut self.base,
&mut self.hooks,
);
}} else {{
self.base.reset();
}}
}}
/// Replaces the input stream and resets runtime and lifecycle state.
pub fn set_input_stream(&mut self, input: I) {{
if H::ENABLES_LEXER_LIFECYCLE {{
antlr4_runtime::atn::lexer::set_input_stream_with_semantic_hooks(
&mut self.base,
&mut self.hooks,
input,
);
}} else {{
self.base.set_input_stream(input);
}}
}}
/// Clears the learned lexer DFA shared by this grammar.
pub fn clear_dfa(&self) {{
self.base.clear_dfa();
}}
{action_method}
{predicate_method}
}}
{typed_lexer_constructor}
impl<I, H> GeneratedLexer for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
}}
impl<I, H> Recognizer for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn data(&self) -> &antlr4_runtime::RecognizerData {{
self.base.data()
}}
fn data_mut(&mut self) -> &mut antlr4_runtime::RecognizerData {{
self.base.data_mut()
}}
}}
impl<I, H> Lexer for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn mode(&self) -> i32 {{ self.base.mode() }}
fn set_mode(&mut self, mode: i32) {{ self.base.set_mode(mode); }}
fn push_mode(&mut self, mode: i32) {{ self.base.push_mode(mode); }}
fn pop_mode(&mut self) -> Option<i32> {{ self.base.pop_mode() }}
}}
impl<I, H> TokenSource for {type_name}<I, H>
where
I: CharStream,
H: antlr4_runtime::SemanticHooks,
{{
fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result<TokenId, TokenStoreError> {{
{next_token_call}
}}
fn line(&self) -> usize {{ self.base.line() }}
fn column(&self) -> usize {{ self.base.column() }}
fn source_name(&self) -> &str {{ self.base.source_name() }}
fn source_text(&self) -> Option<std::rc::Rc<str>> {{ self.base.source_text() }}
fn drain_errors(&mut self) -> Vec<antlr4_runtime::token::TokenSourceError> {{
self.base.drain_errors()
}}
fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool {{
antlr4_runtime::Recognizer::notify_error_listeners(
self,
source_error.line,
source_error.column,
&source_error.message,
None,
);
true
}}
fn lexer_dfa_string(&self) -> String {{
self.base.lexer_dfa_string()
}}
}}
{generated_footer}"#
))
}
fn lexer_actions_require_semantic_hooks(
coordinates: &[(i32, i32)],
rule_names: &[String],
actions: &[((i32, i32), ActionTemplate)],
patterns: &SemPatternFile,
sem_unknown: SemUnknownPolicy,
) -> bool {
coordinates.iter().any(|coordinate| {
let (rule_index, action_index) = *coordinate;
let rule_name = usize::try_from(rule_index)
.ok()
.and_then(|rule| rule_names.get(rule).map(String::as_str));
if let Some(override_) = patterns.coordinate_override(
SemanticsKind::LexerAction,
rule_name,
usize::try_from(action_index).ok(),
None,
) {
return override_.dispose == CoordinateDispose::Hook;
}
if sem_unknown != SemUnknownPolicy::Hook {
return false;
}
!matches!(
actions
.iter()
.find(|(candidate, _)| candidate == coordinate)
.map(|(_, template)| template),
Some(ActionTemplate::LexerPopMode)
)
})
}
/// Compiles the lexer DFA at generation time and flattens it for embedding.
///
/// An empty stream makes the generated lexer fall back to compiling the DFA
/// from its ATN at first use, so generation never fails on this step.
fn compiled_lexer_dfa_words(data: &InterpData) -> String {
if data.atn.is_empty() {
return String::new();
}
let serialized = SerializedAtn::from_i32(&data.atn);
let Ok(atn) = AtnDeserializer::new(&serialized).deserialize() else {
return String::new();
};
let words = CompiledLexerDfa::compile(&atn).serialize();
let rendered: Vec<String> = words.iter().map(u32::to_string).collect();
rendered.join(",")
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct GeneratedParserRule {
rule_index: usize,
entry_state: usize,
left_recursive: bool,
steps: Vec<GeneratedParserStep>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum GeneratedParserStep {
MatchToken {
token_type: i32,
follow_state: usize,
},
MatchSet {
token_set: Option<usize>,
intervals: Vec<(i32, i32)>,
follow_state: usize,
},
MatchNotSet {
token_set: Option<usize>,
intervals: Vec<(i32, i32)>,
follow_state: usize,
},
MatchWildcard {
follow_state: usize,
},
Precedence(i32),
Predicate {
rule_index: usize,
pred_index: usize,
},
Action {
source_state: usize,
rule_index: usize,
},
CallRule {
source_state: usize,
rule_index: usize,
precedence: GeneratedRuleCallPrecedence,
},
Decision {
state: usize,
decision: usize,
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
fast_path: Option<GeneratedDecisionFastPath>,
alts: Vec<Vec<Self>>,
},
StarLoop {
state: usize,
decision: usize,
enter_alt: usize,
exit_alt: usize,
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
/// `true` for a `+` (one-or-more) loop, `false` for a `*` (zero-or-more)
/// loop. A `+` loop's mandatory first element is iteration 1, so its first
/// loop-back sync recovers like ANTLR's `STAR_LOOP_BACK`/`PLUS_LOOP_BACK`
/// (multi-token `consumeUntil`); a `*` loop's first sync is at the entry,
/// which recovers like `STAR_LOOP_ENTRY` (single-token deletion).
plus_loop: bool,
fast_path: Option<GeneratedDecisionFastPath>,
body: Vec<Self>,
},
LeftRecursiveLoop {
state: usize,
decision: usize,
enter_alt: usize,
exit_alt: usize,
rule_index: usize,
entry_state: usize,
body: Vec<Self>,
},
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GeneratedRuleCallPrecedence {
Literal(i32),
InheritLocal,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct GeneratedDecisionFastPath {
arms: Vec<GeneratedDecisionFastArm>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct GeneratedDecisionFastArm {
alt: usize,
intervals: Vec<(i32, i32)>,
}
#[derive(Clone, Copy)]
struct DecisionRender<'a> {
state: usize,
decision: usize,
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
fast_path: Option<&'a GeneratedDecisionFastPath>,
alts: &'a [Vec<GeneratedParserStep>],
}
#[derive(Clone, Copy)]
struct StarLoopRender<'a> {
state: usize,
decision: usize,
alts: (usize, usize),
track_alt_number: bool,
allow_semantic_context: bool,
force_context: bool,
plus_loop: bool,
fast_path: Option<&'a GeneratedDecisionFastPath>,
body: &'a [GeneratedParserStep],
}
#[derive(Clone, Copy)]
struct LeftRecursiveLoopRender<'a> {
state: usize,
decision: usize,
alts: (usize, usize),
rule: (usize, usize),
body: &'a [GeneratedParserStep],
}
/// Embedded-mode data consulted while rendering rule bodies: verbatim
/// action/predicate expressions plus per-rule `@init` / `@after` bodies.
#[derive(Clone, Copy)]
struct EmbeddedStepRender<'a> {
/// Keep every decision on the adaptive simulator (no LL(1)/fast-path
/// shortcuts) regardless of the tool classification.
force_adaptive: bool,
/// Tool-classified non-LL(1) decisions ([`tool_decision_analysis`]).
adaptive_decisions: &'a BTreeSet<usize>,
/// Tool LOOK(1) dispatch intervals for LL(1)-disjoint decisions.
ll1_decision_arms: &'a BTreeMap<usize, Vec<Vec<(i32, i32)>>>,
predicates: &'a BTreeMap<(usize, usize), (String, Option<String>)>,
rule_has_attrs: &'a [bool],
init_entry: &'a BTreeMap<usize, String>,
after: &'a BTreeMap<usize, String>,
call_args: &'a BTreeMap<usize, String>,
rule_arg0: &'a [Option<String>],
}
impl EmbeddedStepRender<'_> {
/// Java routes this decision through `adaptivePredict` — only there are
/// DFA states learned and full-context diagnostics emitted — so the
/// generated code must skip its LL(1)/fast-path shortcuts for it.
fn adaptive_decision(&self, decision: usize) -> bool {
self.force_adaptive || self.adaptive_decisions.contains(&decision)
}
/// Complete dispatch table for a tool-LL(1) decision, exit alternatives
/// included — Java's switch compilation. Legit input never reaches the
/// simulator through this, so no DFA state is ever learned for the
/// decision (matching the dump).
fn tool_ll1_fast_path(&self, decision: usize) -> Option<GeneratedDecisionFastPath> {
let arms = self.ll1_decision_arms.get(&decision)?;
Some(GeneratedDecisionFastPath {
arms: arms
.iter()
.enumerate()
.map(|(index, intervals)| GeneratedDecisionFastArm {
alt: index + 1,
intervals: intervals.clone(),
})
.collect(),
})
}
}
#[derive(Clone, Copy)]
struct PortableLocalStepRender<'a> {
declarations: &'a [Vec<String>],
inline_actions: &'a BTreeMap<usize, String>,
predicates: &'a BTreeMap<(usize, usize), (String, Option<String>)>,
required_generated_rules: &'a BTreeSet<usize>,
}
#[derive(Clone, Copy)]
struct GeneratedStepRenderContext<'a> {
/// `Some` in embedded mode: actions/predicates are verbatim Rust.
embedded: Option<EmbeddedStepRender<'a>>,
/// Portable raw-grammar boolean locals translated without embedded mode.
portable_locals: Option<PortableLocalStepRender<'a>>,
inline_action_statements: &'a BTreeMap<usize, String>,
track_alt_numbers: bool,
direct_generated_rule_calls: &'a [bool],
atn_preferred_rule_calls: &'a [bool],
}
struct GeneratedParserCompileContext<'a> {
atn: &'a ParserAtn,
decision_by_state: &'a [Option<usize>],
rule_args: &'a [(usize, usize, RuleArgTemplate)],
inline_action_states: &'a BTreeSet<usize>,
action_states: &'a BTreeSet<usize>,
generated_action_states: &'a BTreeSet<usize>,
predicate_coordinates: &'a BTreeSet<(usize, usize)>,
generated_predicate_coordinates: &'a BTreeSet<(usize, usize)>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct TypedHookMapping {
rule_index: usize,
pred_index: usize,
method_name: String,
call: SemanticHelperCall,
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
enum LexerTypedHookKind {
Predicate,
Action,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct LexerTypedHookMapping {
rule_index: usize,
coordinate_index: usize,
kind: LexerTypedHookKind,
method_name: String,
call: SemanticHelperCall,
}
#[derive(Clone, Copy, Debug, Default)]
struct ParserRenderOptions<'a> {
require_generated_parser: bool,
/// Splice verbatim Rust action/predicate bodies from the grammar
/// (`--actions embedded`).
embedded: bool,
sem_unknown: SemUnknownPolicy,
patterns: Option<&'a SemPatternFile>,
}
#[derive(Clone, Copy)]
struct ActionStateSets<'a> {
all: &'a BTreeSet<usize>,
generated: &'a BTreeSet<usize>,
inline: &'a BTreeSet<usize>,
}
#[derive(Clone, Copy)]
struct PredicateCoordinateSets<'a> {
all: &'a BTreeSet<(usize, usize)>,
generated: &'a BTreeSet<(usize, usize)>,
}
const fn generated_action_state_sets<'a>(
context: &GeneratedParserCompileContext<'a>,
) -> ActionStateSets<'a> {
ActionStateSets {
all: context.action_states,
generated: context.generated_action_states,
inline: context.inline_action_states,
}
}
const fn generated_predicate_coordinate_sets<'a>(
context: &GeneratedParserCompileContext<'a>,
) -> PredicateCoordinateSets<'a> {
PredicateCoordinateSets {
all: context.predicate_coordinates,
generated: context.generated_predicate_coordinates,
}
}
/// Compiles the parser ATN subset that is safe to emit as recursive-descent
/// Rust today. Unsupported states deliberately return `None` so the generated
/// method can keep using the interpreter fallback until more ATN shapes are
/// covered.
fn parser_generated_rules(
data: &InterpData,
enabled_rules: &[bool],
rule_args: &[(usize, usize, RuleArgTemplate)],
action_states: ActionStateSets<'_>,
predicate_coordinates: PredicateCoordinateSets<'_>,
require_generated_callees: bool,
) -> io::Result<Vec<Option<GeneratedParserRule>>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let decision_by_state = decision_by_state(&atn);
let context = GeneratedParserCompileContext {
atn: &atn,
decision_by_state: &decision_by_state,
rule_args,
inline_action_states: action_states.inline,
action_states: action_states.all,
generated_action_states: action_states.generated,
predicate_coordinates: predicate_coordinates.all,
generated_predicate_coordinates: predicate_coordinates.generated,
};
let mut rules = (0..data.rule_names.len())
.map(|rule_index| {
if enabled_rules.get(rule_index).copied().unwrap_or_default() {
compile_generated_parser_rule(&context, rule_index)
} else {
None
}
})
.collect::<Vec<_>>();
if require_generated_callees {
drop_rules_calling_disabled_rules(&mut rules);
}
Ok(rules)
}
fn drop_rules_calling_disabled_rules(rules: &mut [Option<GeneratedParserRule>]) {
loop {
let enabled = rules.iter().map(Option::is_some).collect::<Vec<_>>();
let drop_index = rules.iter().filter_map(Option::as_ref).find_map(|rule| {
generated_steps_call_disabled_rule(&rule.steps, &enabled).then_some(rule.rule_index)
});
let Some(rule_index) = drop_index else {
return;
};
rules[rule_index] = None;
}
}
const ATN_PREFERRED_LEADING_CALL_CHAIN_MIN: usize = 8;
const ATN_PREFERRED_CHAIN_MIN_DECISION_DENSITY_NUMERATOR: usize = 2;
const ATN_PREFERRED_WRAPPER_MIN_DECISION_COST: usize = 2;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct GeneratedRuleShape {
decision_cost: usize,
action_or_predicate_count: usize,
}
impl AddAssign for GeneratedRuleShape {
fn add_assign(&mut self, rhs: Self) {
self.decision_cost += rhs.decision_cost;
self.action_or_predicate_count += rhs.action_or_predicate_count;
}
}
#[cfg(test)]
fn generated_atn_preferred_rule_calls(
rules: &[Option<GeneratedParserRule>],
rule_names: &[String],
) -> Vec<bool> {
generated_atn_preferred_rule_calls_excluding(rules, rule_names, &BTreeSet::new())
}
fn generated_atn_preferred_rule_calls_excluding(
rules: &[Option<GeneratedParserRule>],
_rule_names: &[String],
force_generated: &BTreeSet<usize>,
) -> Vec<bool> {
let leading_rule_calls = rules
.iter()
.map(|rule| {
rule.as_ref()
.and_then(|rule| generated_steps_leading_mandatory_rule_call(&rule.steps))
})
.collect::<Vec<_>>();
let shapes = rules
.iter()
.map(|rule| {
rule.as_ref()
.map_or_else(GeneratedRuleShape::default, generated_rule_shape)
})
.collect::<Vec<_>>();
let mut preferred = vec![false; rules.len()];
for start in 0..rules.len() {
if rules[start].is_none() {
continue;
}
let mut chain = Vec::new();
let mut seen = vec![false; rules.len()];
let mut current = start;
loop {
if current >= rules.len() || rules[current].is_none() || seen[current] {
break;
}
seen[current] = true;
chain.push(current);
let Some(next) = leading_rule_calls[current] else {
break;
};
current = next;
}
if chain.len() >= ATN_PREFERRED_LEADING_CALL_CHAIN_MIN
&& generated_atn_preferred_chain_is_expensive(&chain, &shapes)
{
for rule_index in chain {
preferred[rule_index] = true;
}
}
}
propagate_atn_preferred_wrappers(rules, &shapes, &mut preferred);
for rule_index in force_generated {
if let Some(entry) = preferred.get_mut(*rule_index) {
*entry = false;
}
}
preferred
}
fn generated_rule_callers_reaching(
rules: &[Option<GeneratedParserRule>],
target_rules: &BTreeSet<usize>,
) -> BTreeSet<usize> {
let mut reaching = target_rules.clone();
loop {
let mut changed = false;
for rule in rules.iter().flatten() {
if reaching.contains(&rule.rule_index)
|| !generated_steps_call_any_rule(&rule.steps, &reaching)
{
continue;
}
changed |= reaching.insert(rule.rule_index);
}
if !changed {
return reaching;
}
}
}
fn parser_rule_callers_reaching(
data: &InterpData,
target_rules: &BTreeSet<usize>,
) -> io::Result<BTreeSet<usize>> {
if target_rules.is_empty() {
return Ok(BTreeSet::new());
}
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
Ok(atn_rule_callers_reaching(
&atn,
target_rules,
data.rule_names.len(),
))
}
fn atn_rule_callers_reaching(
atn: &ParserAtn,
target_rules: &BTreeSet<usize>,
rule_count: usize,
) -> BTreeSet<usize> {
let mut reaching = target_rules.clone();
loop {
let mut changed = false;
for state in atn.states() {
let Some(caller_rule) = state.rule_index().filter(|index| *index < rule_count) else {
continue;
};
if reaching.contains(&caller_rule) {
continue;
}
let calls_reaching_rule = state.transitions().iter().any(|transition| {
matches!(
transition.data(),
ParserTransitionData::Rule { rule_index, .. }
if reaching.contains(&rule_index)
)
});
if calls_reaching_rule {
changed |= reaching.insert(caller_rule);
}
}
if !changed {
return reaching;
}
}
}
fn generated_steps_call_any_rule(
steps: &[GeneratedParserStep],
rule_indices: &BTreeSet<usize>,
) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::CallRule { rule_index, .. } => rule_indices.contains(rule_index),
GeneratedParserStep::Decision { alts, .. } => alts
.iter()
.any(|alt| generated_steps_call_any_rule(alt, rule_indices)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
generated_steps_call_any_rule(body, rule_indices)
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => false,
})
}
fn generated_atn_preferred_chain_is_expensive(
chain: &[usize],
shapes: &[GeneratedRuleShape],
) -> bool {
let decision_cost = chain
.iter()
.filter_map(|rule_index| shapes.get(*rule_index))
.map(|shape| shape.decision_cost)
.sum::<usize>();
decision_cost >= chain.len() * ATN_PREFERRED_CHAIN_MIN_DECISION_DENSITY_NUMERATOR
}
fn propagate_atn_preferred_wrappers(
rules: &[Option<GeneratedParserRule>],
shapes: &[GeneratedRuleShape],
preferred: &mut [bool],
) {
loop {
let mut changed = false;
for (rule_index, rule) in rules.iter().enumerate() {
if preferred.get(rule_index).copied().unwrap_or_default() {
continue;
}
let Some(rule) = rule else {
continue;
};
if !generated_rule_is_atn_preferred_wrapper(rule, shapes, preferred) {
continue;
}
preferred[rule_index] = true;
changed = true;
}
if !changed {
return;
}
}
}
fn generated_rule_is_atn_preferred_wrapper(
rule: &GeneratedParserRule,
shapes: &[GeneratedRuleShape],
preferred: &[bool],
) -> bool {
if rule.left_recursive {
return false;
}
let shape = shapes.get(rule.rule_index).copied().unwrap_or_default();
shape.action_or_predicate_count == 0
&& shape.decision_cost >= ATN_PREFERRED_WRAPPER_MIN_DECISION_COST
&& generated_steps_call_atn_preferred_rule(&rule.steps, preferred)
}
fn generated_rule_shape(rule: &GeneratedParserRule) -> GeneratedRuleShape {
generated_steps_shape(&rule.steps)
}
fn generated_steps_shape(steps: &[GeneratedParserStep]) -> GeneratedRuleShape {
let mut shape = GeneratedRuleShape::default();
for step in steps {
shape += generated_step_shape(step);
}
shape
}
fn generated_step_shape(step: &GeneratedParserStep) -> GeneratedRuleShape {
match step {
GeneratedParserStep::Decision {
allow_semantic_context,
force_context,
fast_path,
alts,
..
} => {
let mut shape = GeneratedRuleShape {
decision_cost: usize::from(
fast_path.is_none() || *allow_semantic_context || *force_context,
),
action_or_predicate_count: 0,
};
for alt in alts {
shape += generated_steps_shape(alt);
}
shape
}
GeneratedParserStep::StarLoop {
allow_semantic_context,
force_context,
fast_path,
body,
..
} => {
let mut shape = GeneratedRuleShape {
decision_cost: usize::from(
fast_path.is_none() || *allow_semantic_context || *force_context,
),
action_or_predicate_count: 0,
};
shape += generated_steps_shape(body);
shape
}
GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
let mut shape = GeneratedRuleShape {
decision_cost: 1,
action_or_predicate_count: 0,
};
shape += generated_steps_shape(body);
shape
}
GeneratedParserStep::Predicate { .. } | GeneratedParserStep::Action { .. } => {
GeneratedRuleShape {
decision_cost: 0,
action_or_predicate_count: 1,
}
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::CallRule { .. } => GeneratedRuleShape::default(),
}
}
fn generated_steps_call_atn_preferred_rule(
steps: &[GeneratedParserStep],
preferred: &[bool],
) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::CallRule { rule_index, .. } => {
preferred.get(*rule_index).copied().unwrap_or_default()
}
GeneratedParserStep::Decision { alts, .. } => alts
.iter()
.any(|alt| generated_steps_call_atn_preferred_rule(alt, preferred)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
generated_steps_call_atn_preferred_rule(body, preferred)
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => false,
})
}
fn generated_steps_leading_mandatory_rule_call(steps: &[GeneratedParserStep]) -> Option<usize> {
for step in steps {
match step {
GeneratedParserStep::CallRule { rule_index, .. } => return Some(*rule_index),
GeneratedParserStep::Decision { alts, .. } if generated_alts_are_nullable(alts) => {}
GeneratedParserStep::Decision { alts, .. } => {
return generated_alts_common_leading_mandatory_rule_call(alts);
}
GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => {}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. } => return None,
}
}
None
}
fn generated_alts_common_leading_mandatory_rule_call(
alts: &[Vec<GeneratedParserStep>],
) -> Option<usize> {
let mut common = None;
for alt in alts {
let rule_index = generated_steps_leading_mandatory_rule_call(alt)?;
match common {
Some(common_rule_index) if common_rule_index != rule_index => return None,
Some(_) => {}
None => common = Some(rule_index),
}
}
common
}
fn generated_alts_are_nullable(alts: &[Vec<GeneratedParserStep>]) -> bool {
alts.iter().any(|alt| generated_steps_are_nullable(alt))
}
fn generated_steps_are_nullable(steps: &[GeneratedParserStep]) -> bool {
steps.iter().all(generated_step_is_nullable)
}
fn generated_step_is_nullable(step: &GeneratedParserStep) -> bool {
match step {
GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => true,
GeneratedParserStep::Decision { alts, .. } => generated_alts_are_nullable(alts),
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. } => false,
}
}
fn require_all_parser_rules_generated(
rules: &[Option<GeneratedParserRule>],
data: &InterpData,
) -> io::Result<()> {
let missing = rules
.iter()
.enumerate()
.filter(|(_, rule)| rule.is_none())
.map(|(index, _)| {
data.rule_names
.get(index)
.map_or_else(|| index.to_string(), Clone::clone)
})
.collect::<Vec<_>>();
if missing.is_empty() {
return Ok(());
}
Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"generated parser did not emit {} rule(s): {}",
missing.len(),
missing.join(", ")
),
))
}
fn require_portable_local_rules_generated(
rules: &[Option<GeneratedParserRule>],
required: &BTreeSet<usize>,
data: &InterpData,
) -> io::Result<()> {
let missing = required
.iter()
.filter(|rule_index| rules.get(**rule_index).is_none_or(Option::is_none))
.map(|rule_index| {
data.rule_names
.get(*rule_index)
.map_or_else(|| rule_index.to_string(), Clone::clone)
})
.collect::<Vec<_>>();
if missing.is_empty() {
return Ok(());
}
Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"portable local semantics require {} generated parser rule(s): {}",
missing.len(),
missing.join(", ")
),
))
}
fn generated_steps_call_disabled_rule(steps: &[GeneratedParserStep], enabled: &[bool]) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::CallRule { rule_index, .. } => {
!enabled.get(*rule_index).copied().unwrap_or_default()
}
GeneratedParserStep::Decision { alts, .. } => alts
.iter()
.any(|alt| generated_steps_call_disabled_rule(alt, enabled)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
generated_steps_call_disabled_rule(body, enabled)
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => false,
})
}
fn decision_by_state(atn: &ParserAtn) -> Vec<Option<usize>> {
let mut decision_by_state = vec![None; atn.state_count()];
for (decision, state_number) in atn.decision_to_state().iter().enumerate() {
if let Some(slot) = decision_by_state.get_mut(state_number) {
*slot = Some(decision);
}
}
decision_by_state
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
struct GeneratedLookSet {
symbols: BTreeSet<i32>,
nullable: bool,
}
#[derive(Default)]
struct GeneratedFirstSetCtx {
cache: BTreeMap<(usize, usize), GeneratedLookSet>,
in_progress: BTreeSet<(usize, usize)>,
hit_cycle: bool,
}
fn generated_decision_fast_path<'a>(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
alts: impl IntoIterator<Item = (usize, &'a [GeneratedParserStep])>,
) -> Option<GeneratedDecisionFastPath> {
if state.precedence_rule_decision() || state.non_greedy() {
return None;
}
let mut first_ctx = GeneratedFirstSetCtx::default();
let mut symbol_alts = BTreeMap::<i32, Option<usize>>::new();
for (alt, steps) in alts {
let look = generated_steps_first_set(context.atn, steps, &mut first_ctx);
if look.nullable {
return None;
}
for symbol in look.symbols {
match symbol_alts.get(&symbol).copied().flatten() {
None if symbol_alts.contains_key(&symbol) => {}
None => {
symbol_alts.insert(symbol, Some(alt));
}
Some(existing) if existing == alt => {}
Some(_) => {
symbol_alts.insert(symbol, None);
}
}
}
}
let mut symbols_by_alt = BTreeMap::<usize, BTreeSet<i32>>::new();
for (symbol, alt) in symbol_alts {
if let Some(alt) = alt {
symbols_by_alt.entry(alt).or_default().insert(symbol);
}
}
let arms = symbols_by_alt
.into_iter()
.map(|(alt, symbols)| GeneratedDecisionFastArm {
alt,
intervals: symbols_to_ranges(symbols),
})
.filter(|arm| !arm.intervals.is_empty())
.collect::<Vec<_>>();
(!arms.is_empty()).then_some(GeneratedDecisionFastPath { arms })
}
fn generated_steps_first_set(
atn: &ParserAtn,
steps: &[GeneratedParserStep],
ctx: &mut GeneratedFirstSetCtx,
) -> GeneratedLookSet {
let mut first = GeneratedLookSet::default();
for step in steps {
match step {
GeneratedParserStep::MatchToken { token_type, .. } => {
first.symbols.insert(*token_type);
first.nullable = false;
return first;
}
GeneratedParserStep::MatchSet { intervals, .. } => {
for (start, stop) in intervals {
first.symbols.extend(*start..=*stop);
}
first.nullable = false;
return first;
}
GeneratedParserStep::MatchNotSet { intervals, .. } => {
first.symbols.extend(1..=atn.max_token_type());
for (start, stop) in intervals {
for symbol in *start..=*stop {
first.symbols.remove(&symbol);
}
}
first.nullable = false;
return first;
}
GeneratedParserStep::MatchWildcard { .. } => {
first.symbols.extend(1..=atn.max_token_type());
first.nullable = false;
return first;
}
GeneratedParserStep::CallRule { rule_index, .. } => {
let Some(start) = atn.rule_to_start_state().get(*rule_index) else {
return GeneratedLookSet::default();
};
let Some(stop) = atn.rule_to_stop_state().get(*rule_index) else {
return GeneratedLookSet::default();
};
let child = generated_rule_first_set(atn, start, stop, ctx);
first.symbols.extend(child.symbols);
if !child.nullable {
first.nullable = false;
return first;
}
}
GeneratedParserStep::Decision { alts, .. } => {
let nested = generated_alt_steps_first_set(atn, alts, ctx);
first.symbols.extend(nested.symbols);
if !nested.nullable {
first.nullable = false;
return first;
}
}
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
let nested = generated_steps_first_set(atn, body, ctx);
first.symbols.extend(nested.symbols);
}
GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. } => {}
}
}
first.nullable = true;
first
}
fn generated_alt_steps_first_set(
atn: &ParserAtn,
alts: &[Vec<GeneratedParserStep>],
ctx: &mut GeneratedFirstSetCtx,
) -> GeneratedLookSet {
let mut first = GeneratedLookSet::default();
for alt in alts {
let alt_first = generated_steps_first_set(atn, alt, ctx);
first.symbols.extend(alt_first.symbols);
first.nullable |= alt_first.nullable;
}
first
}
fn generated_rule_first_set(
atn: &ParserAtn,
state_number: usize,
rule_stop_state: usize,
ctx: &mut GeneratedFirstSetCtx,
) -> GeneratedLookSet {
let key = (state_number, rule_stop_state);
if let Some(cached) = ctx.cache.get(&key) {
return cached.clone();
}
if !ctx.in_progress.insert(key) {
return GeneratedLookSet::default();
}
let saved_hit_cycle = ctx.hit_cycle;
ctx.hit_cycle = false;
let mut first = GeneratedLookSet::default();
generated_rule_first_set_inner(
atn,
state_number,
rule_stop_state,
ctx,
&mut BTreeSet::new(),
&mut first,
);
ctx.in_progress.remove(&key);
if !ctx.hit_cycle {
ctx.cache.insert(key, first.clone());
}
ctx.hit_cycle = saved_hit_cycle || ctx.hit_cycle;
first
}
fn generated_rule_first_set_inner(
atn: &ParserAtn,
state_number: usize,
rule_stop_state: usize,
ctx: &mut GeneratedFirstSetCtx,
visited: &mut BTreeSet<usize>,
first: &mut GeneratedLookSet,
) {
if !visited.insert(state_number) {
return;
}
if state_number == rule_stop_state {
first.nullable = true;
return;
}
let Some(state) = atn.state(state_number) else {
return;
};
for transition in state.transitions() {
let symbols = generated_transition_symbols(transition, atn.max_token_type());
if !symbols.is_empty() {
first.symbols.extend(symbols);
continue;
}
match transition.data() {
ParserTransitionData::Epsilon { target }
| ParserTransitionData::Action { target, .. }
| ParserTransitionData::Predicate { target, .. }
| ParserTransitionData::Precedence { target, .. } => {
generated_rule_first_set_inner(atn, target, rule_stop_state, ctx, visited, first);
}
ParserTransitionData::Rule {
target,
rule_index,
follow_state,
..
} => {
let Some(child_stop) = atn.rule_to_stop_state().get(rule_index) else {
continue;
};
let child_key = (target, child_stop);
if ctx.in_progress.contains(&child_key) && !ctx.cache.contains_key(&child_key) {
ctx.hit_cycle = true;
}
let child = generated_rule_first_set(atn, target, child_stop, ctx);
first.symbols.extend(child.symbols);
if child.nullable {
generated_rule_first_set_inner(
atn,
follow_state,
rule_stop_state,
ctx,
visited,
first,
);
}
}
ParserTransitionData::Atom { .. }
| ParserTransitionData::Range { .. }
| ParserTransitionData::Set { .. }
| ParserTransitionData::NotSet { .. }
| ParserTransitionData::Wildcard { .. } => {}
}
}
}
fn generated_transition_symbols(
transition: ParserTransition<'_>,
max_token_type: i32,
) -> BTreeSet<i32> {
let mut symbols = BTreeSet::new();
match transition.data() {
ParserTransitionData::Atom { label, .. } => {
symbols.insert(label);
}
ParserTransitionData::Range { start, stop, .. } => {
symbols.extend(start..=stop);
}
ParserTransitionData::Set { set, .. } => {
for (start, stop) in set.ranges() {
symbols.extend(start..=stop);
}
}
ParserTransitionData::NotSet { set, .. } => {
symbols.extend((1..=max_token_type).filter(|symbol| !set.contains(*symbol)));
}
ParserTransitionData::Wildcard { .. } => {
symbols.extend(1..=max_token_type);
}
ParserTransitionData::Epsilon { .. }
| ParserTransitionData::Rule { .. }
| ParserTransitionData::Predicate { .. }
| ParserTransitionData::Action { .. }
| ParserTransitionData::Precedence { .. } => {}
}
symbols
}
fn symbols_to_ranges(symbols: BTreeSet<i32>) -> Vec<(i32, i32)> {
let mut ranges = Vec::new();
for symbol in symbols {
match ranges.last_mut() {
Some((_, stop)) if *stop + 1 == symbol => *stop = symbol,
_ => ranges.push((symbol, symbol)),
}
}
ranges
}
fn state_tracks_alt_number(state: ParserAtnState<'_>) -> bool {
matches!(
state.kind(),
AtnStateKind::Basic
| AtnStateKind::BlockStart
| AtnStateKind::PlusBlockStart
| AtnStateKind::StarBlockStart
| AtnStateKind::StarLoopEntry
) && !state.precedence_rule_decision()
&& state.transitions().len() > 1
}
fn compile_generated_parser_rule(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
) -> Option<GeneratedParserRule> {
let entry_state = context.atn.rule_to_start_state().get(rule_index)?;
let stop_state = context.atn.rule_to_stop_state().get(rule_index)?;
let start = context.atn.state(entry_state)?;
if start.left_recursive_rule() {
return compile_generated_left_recursive_parser_rule(
context,
rule_index,
entry_state,
stop_state,
);
}
let mut visited = BTreeSet::new();
let steps = compile_generated_parser_path(context, entry_state, stop_state, &mut visited)?;
Some(GeneratedParserRule {
rule_index,
entry_state,
left_recursive: false,
steps,
})
}
fn compile_generated_left_recursive_parser_rule(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
entry_state: usize,
stop_state: usize,
) -> Option<GeneratedParserRule> {
let loop_entry = find_left_recursive_loop_entry(context, rule_index)?;
let mut visited = BTreeSet::new();
let mut steps = compile_generated_parser_path(context, entry_state, loop_entry, &mut visited)?;
let loop_state = context.atn.state(loop_entry)?;
let decision = context
.decision_by_state
.get(loop_entry)
.copied()
.flatten()?;
let (loop_step, exit_target) = compile_generated_left_recursive_loop(
context,
rule_index,
entry_state,
loop_state,
decision,
)?;
steps.push(loop_step);
steps.extend(compile_generated_parser_path(
context,
exit_target,
stop_state,
&mut BTreeSet::new(),
)?);
Some(GeneratedParserRule {
rule_index,
entry_state,
left_recursive: true,
steps,
})
}
fn find_left_recursive_loop_entry(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
) -> Option<usize> {
context.atn.states().find_map(|state| {
(state.rule_index() == Some(rule_index)
&& state.kind() == AtnStateKind::StarLoopEntry
&& state.precedence_rule_decision())
.then_some(state.state_number())
})
}
fn compile_generated_left_recursive_loop(
context: &GeneratedParserCompileContext<'_>,
rule_index: usize,
entry_state: usize,
state: ParserAtnState<'_>,
decision: usize,
) -> Option<(GeneratedParserStep, usize)> {
let mut enter = None;
let mut exit = None;
for (index, transition) in state.transitions().iter().enumerate() {
let alt = index + 1;
let target = transition.target();
let target_state = context.atn.state(target)?;
if target_state.kind() == AtnStateKind::LoopEnd {
exit = Some((alt, transition, target, target_state.loop_back_state()?));
} else {
enter = Some((alt, transition));
}
}
let (enter_alt, enter_transition) = enter?;
let (exit_alt, exit_transition, exit_target, loop_back_state) = exit?;
let (enter_step, enter_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
enter_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut body = enter_step.into_iter().collect::<Vec<_>>();
body.extend(compile_generated_parser_path(
context,
enter_target,
loop_back_state,
&mut BTreeSet::new(),
)?);
allow_semantic_context_in_decisions(&mut body);
if !steps_may_consume(&body) {
return None;
}
let (exit_step, _) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
exit_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
if exit_step.is_some() {
return None;
}
Some((
GeneratedParserStep::LeftRecursiveLoop {
state: state.state_number(),
decision,
enter_alt,
exit_alt,
rule_index,
entry_state,
body,
},
exit_target,
))
}
fn compile_generated_parser_path(
context: &GeneratedParserCompileContext<'_>,
state_number: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
if state_number == stop_state {
return Some(Vec::new());
}
if !visited.insert(state_number) {
return None;
}
let state = context.atn.state(state_number)?;
let steps = if let Some(decision) = context
.decision_by_state
.get(state_number)
.copied()
.flatten()
{
compile_generated_parser_decision_state(context, state, decision, stop_state, visited)?
} else {
let transition = state.transitions().first()?;
if state.transitions().len() != 1 {
return None;
}
let (step, target) = compile_generated_parser_transition(
state_number,
context.rule_args,
transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut steps = step.into_iter().collect::<Vec<_>>();
steps.extend(compile_generated_parser_path(
context, target, stop_state, visited,
)?);
steps
};
visited.remove(&state_number);
Some(steps)
}
fn compile_generated_parser_decision_state(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
match state.kind() {
AtnStateKind::BlockStart | AtnStateKind::PlusBlockStart | AtnStateKind::StarBlockStart => {
compile_generated_parser_block_decision(context, state, decision, stop_state, visited)
}
AtnStateKind::StarLoopEntry => {
compile_generated_parser_star_loop(context, state, decision, stop_state, visited)
}
AtnStateKind::PlusLoopBack => {
compile_generated_parser_plus_loop(context, state, decision, stop_state, visited)
}
_ => None,
}
}
fn compile_generated_parser_block_decision(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
let end_state = state.end_state()?;
let mut alts = Vec::with_capacity(state.transitions().len());
for transition in state.transitions() {
let (step, target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut alt_visited = visited.clone();
let mut alt_steps = step.into_iter().collect::<Vec<_>>();
alt_steps.extend(compile_generated_parser_path(
context,
target,
end_state,
&mut alt_visited,
)?);
alts.push(alt_steps);
}
let mut steps = vec![GeneratedParserStep::Decision {
state: state.state_number(),
decision,
track_alt_number: state_tracks_alt_number(state),
allow_semantic_context: alts.iter().any(|alt| steps_contain_predicate(alt)),
force_context: state.non_greedy(),
fast_path: generated_decision_fast_path(
context,
state,
alts.iter()
.enumerate()
.map(|(index, alt)| (index + 1, alt.as_slice())),
),
alts,
}];
steps.extend(compile_generated_parser_path(
context, end_state, stop_state, visited,
)?);
Some(steps)
}
fn compile_generated_parser_star_loop(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
let mut enter = None;
let mut exit = None;
for (index, transition) in state.transitions().iter().enumerate() {
let alt = index + 1;
let target = transition.target();
let target_state = context.atn.state(target)?;
let target_kind = target_state.kind();
if target_kind == AtnStateKind::LoopEnd {
exit = Some((alt, transition, target_state.loop_back_state()?));
} else {
enter = Some((alt, transition));
}
}
let (enter_alt, enter_transition) = enter?;
let (exit_alt, exit_transition, loop_back_state) = exit?;
let (enter_step, enter_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
enter_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut body_visited = BTreeSet::new();
let mut body = enter_step.into_iter().collect::<Vec<_>>();
body.extend(compile_generated_parser_path(
context,
enter_target,
loop_back_state,
&mut body_visited,
)?);
if !steps_may_consume(&body) {
return None;
}
let (exit_step, exit_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
exit_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
if exit_step.is_some() {
return None;
}
let mut steps = vec![GeneratedParserStep::StarLoop {
state: state.state_number(),
decision,
enter_alt,
exit_alt,
track_alt_number: state_tracks_alt_number(state),
allow_semantic_context: steps_contain_predicate(&body),
force_context: state.non_greedy(),
plus_loop: false,
fast_path: None,
body,
}];
steps.extend(compile_generated_parser_path(
context,
exit_target,
stop_state,
visited,
)?);
Some(steps)
}
fn compile_generated_parser_plus_loop(
context: &GeneratedParserCompileContext<'_>,
state: ParserAtnState<'_>,
decision: usize,
stop_state: usize,
visited: &mut BTreeSet<usize>,
) -> Option<Vec<GeneratedParserStep>> {
let mut enter = None;
let mut exit = None;
for (index, transition) in state.transitions().iter().enumerate() {
let alt = index + 1;
let target = transition.target();
let target_state = context.atn.state(target)?;
if target_state.kind() == AtnStateKind::LoopEnd {
exit = Some((alt, transition));
} else {
enter = Some((alt, transition));
}
}
let (enter_alt, enter_transition) = enter?;
let (enter_step, enter_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
enter_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
let mut body_visited = BTreeSet::new();
let mut body = enter_step.into_iter().collect::<Vec<_>>();
body.extend(compile_generated_parser_path(
context,
enter_target,
state.state_number(),
&mut body_visited,
)?);
if !steps_may_consume(&body) {
return None;
}
let (exit_alt, exit_transition) = exit?;
let (exit_step, exit_target) = compile_generated_parser_transition(
state.state_number(),
context.rule_args,
exit_transition,
generated_action_state_sets(context),
generated_predicate_coordinate_sets(context),
)?;
if exit_step.is_some() {
return None;
}
let mut steps = vec![GeneratedParserStep::StarLoop {
state: state.state_number(),
decision,
enter_alt,
exit_alt,
track_alt_number: state_tracks_alt_number(state),
allow_semantic_context: steps_contain_predicate(&body),
force_context: state.non_greedy(),
plus_loop: true,
fast_path: None,
body,
}];
steps.extend(compile_generated_parser_path(
context,
exit_target,
stop_state,
visited,
)?);
Some(steps)
}
fn steps_may_consume(steps: &[GeneratedParserStep]) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. } => true,
GeneratedParserStep::Action { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. } => false,
GeneratedParserStep::Decision { alts, .. } => alts.iter().any(|alt| steps_may_consume(alt)),
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => steps_may_consume(body),
})
}
fn allow_semantic_context_in_decisions(steps: &mut [GeneratedParserStep]) {
for step in steps {
match step {
GeneratedParserStep::Decision {
allow_semantic_context,
fast_path,
alts,
..
} => {
*allow_semantic_context = true;
*fast_path = None;
for alt in alts {
allow_semantic_context_in_decisions(alt);
}
}
GeneratedParserStep::StarLoop {
allow_semantic_context,
fast_path,
body,
..
} => {
*allow_semantic_context = true;
*fast_path = None;
allow_semantic_context_in_decisions(body);
}
GeneratedParserStep::LeftRecursiveLoop { body, .. } => {
allow_semantic_context_in_decisions(body);
}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::CallRule { .. } => {}
}
}
}
fn steps_contain_predicate(steps: &[GeneratedParserStep]) -> bool {
steps.iter().any(|step| match step {
GeneratedParserStep::Predicate { .. } => true,
GeneratedParserStep::Decision { alts, .. } => {
alts.iter().any(|alt| steps_contain_predicate(alt))
}
GeneratedParserStep::StarLoop { body, .. }
| GeneratedParserStep::LeftRecursiveLoop { body, .. } => steps_contain_predicate(body),
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::Precedence(_)
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::CallRule { .. } => false,
})
}
fn generated_rule_call_precedence(
rule_args: &[(usize, usize, RuleArgTemplate)],
source_state: usize,
rule_index: usize,
transition_precedence: i32,
) -> Option<GeneratedRuleCallPrecedence> {
let Some((_, _, arg)) = rule_args
.iter()
.find(|(arg_source, arg_rule, _)| *arg_source == source_state && *arg_rule == rule_index)
else {
return Some(GeneratedRuleCallPrecedence::Literal(transition_precedence));
};
match arg {
RuleArgTemplate::Literal(value) => i32::try_from(*value)
.ok()
.map(GeneratedRuleCallPrecedence::Literal),
RuleArgTemplate::InheritLocal => Some(GeneratedRuleCallPrecedence::InheritLocal),
}
}
fn compile_generated_parser_transition(
source_state: usize,
rule_args: &[(usize, usize, RuleArgTemplate)],
transition: ParserTransition<'_>,
action_states: ActionStateSets<'_>,
predicate_coordinates: PredicateCoordinateSets<'_>,
) -> Option<(Option<GeneratedParserStep>, usize)> {
match transition.data() {
ParserTransitionData::Epsilon { target } => Some((None, target)),
ParserTransitionData::Atom { target, label } => Some((
Some(GeneratedParserStep::MatchToken {
token_type: label,
follow_state: target,
}),
target,
)),
ParserTransitionData::Range {
target,
start,
stop,
} => Some((
Some(GeneratedParserStep::MatchSet {
token_set: None,
intervals: vec![(start, stop)],
follow_state: target,
}),
target,
)),
ParserTransitionData::Set { target, set } => Some((
Some(GeneratedParserStep::MatchSet {
token_set: (set.kind() == ParserTokenSetKind::Dense).then_some(set.index()),
intervals: set.ranges().collect(),
follow_state: target,
}),
target,
)),
ParserTransitionData::NotSet { target, set } => Some((
Some(GeneratedParserStep::MatchNotSet {
token_set: (set.kind() == ParserTokenSetKind::Dense).then_some(set.index()),
intervals: set.ranges().collect(),
follow_state: target,
}),
target,
)),
ParserTransitionData::Wildcard { target } => Some((
Some(GeneratedParserStep::MatchWildcard {
follow_state: target,
}),
target,
)),
ParserTransitionData::Rule {
rule_index,
follow_state,
precedence,
..
} => Some((
Some(GeneratedParserStep::CallRule {
source_state,
rule_index,
precedence: generated_rule_call_precedence(
rule_args,
source_state,
rule_index,
precedence,
)?,
}),
follow_state,
)),
ParserTransitionData::Action {
target, rule_index, ..
} if action_states.generated.contains(&source_state) => Some((
Some(GeneratedParserStep::Action {
source_state,
rule_index,
}),
target,
)),
ParserTransitionData::Action {
target,
action_index: None,
..
} if !action_states.all.contains(&source_state) => Some((None, target)),
ParserTransitionData::Predicate {
target,
rule_index,
pred_index,
..
} if predicate_coordinates
.generated
.contains(&(rule_index, pred_index)) =>
{
Some((
Some(GeneratedParserStep::Predicate {
rule_index,
pred_index,
}),
target,
))
}
ParserTransitionData::Predicate {
rule_index,
pred_index,
..
} if predicate_coordinates
.all
.contains(&(rule_index, pred_index)) =>
{
None
}
ParserTransitionData::Predicate { target, .. } => Some((None, target)),
ParserTransitionData::Precedence { target, precedence } => {
Some((Some(GeneratedParserStep::Precedence(precedence)), target))
}
ParserTransitionData::Action { .. } => None,
}
}
#[cfg(test)]
fn render_generated_rule_dispatch(
rules: &[Option<GeneratedParserRule>],
direct_generated_rule_calls: &[bool],
inline_action_statements: &BTreeMap<usize, String>,
track_alt_numbers: bool,
) -> String {
render_generated_rule_dispatch_with_rule_names(
rules,
direct_generated_rule_calls,
&[],
inline_action_statements,
track_alt_numbers,
None,
None,
)
}
#[allow(clippy::too_many_arguments)]
fn render_generated_rule_dispatch_with_rule_names(
rules: &[Option<GeneratedParserRule>],
direct_generated_rule_calls: &[bool],
rule_names: &[String],
inline_action_statements: &BTreeMap<usize, String>,
track_alt_numbers: bool,
embedded: Option<EmbeddedStepRender<'_>>,
portable_locals: Option<PortableLocalStepRender<'_>>,
) -> String {
let mut out = String::new();
let force_generated_rules = if embedded.is_some() {
rules.iter().flatten().map(|rule| rule.rule_index).collect()
} else {
portable_locals.map_or_else(BTreeSet::new, |portable| {
generated_rule_callers_reaching(rules, portable.required_generated_rules)
})
};
let atn_preferred_rule_calls =
generated_atn_preferred_rule_calls_excluding(rules, rule_names, &force_generated_rules);
writeln!(
out,
" #[allow(dead_code)]\n fn parse_generated_rule(&mut self, rule_index: usize, precedence: i32, allow_fallback: bool) -> Option<Result<antlr4_runtime::ParseTree, GeneratedRuleError>> {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " let _ = precedence;").expect("writing to a string cannot fail");
writeln!(out, " let _ = allow_fallback;").expect("writing to a string cannot fail");
writeln!(out, " match rule_index {{").expect("writing to a string cannot fail");
for rule in rules.iter().flatten() {
let index = rule.rule_index;
if atn_preferred_rule_calls
.get(index)
.copied()
.unwrap_or_default()
{
writeln!(
out,
" {index} if self.generated_only() => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback)),"
)
.expect("writing to a string cannot fail");
} else {
writeln!(
out,
" {index} => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback)),"
)
.expect("writing to a string cannot fail");
}
}
writeln!(out, " _ => None,").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
let step_render_context = GeneratedStepRenderContext {
embedded,
portable_locals,
inline_action_statements,
track_alt_numbers,
direct_generated_rule_calls,
atn_preferred_rule_calls: &atn_preferred_rule_calls,
};
for rule in rules.iter().flatten() {
let index = rule.rule_index;
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
if rule.left_recursive {
writeln!(
out,
" self.parse_generated_rule_{index}_precedence(precedence, allow_fallback)"
)
.expect("writing to a string cannot fail");
} else {
writeln!(out, " let _ = precedence;").expect("writing to a string cannot fail");
writeln!(
out,
" self.parse_generated_rule_{index}(precedence, allow_fallback)"
)
.expect("writing to a string cannot fail");
}
writeln!(out, " }}").expect("writing to a string cannot fail");
render_generated_rule_method(&mut out, rule, step_render_context);
}
out
}
fn render_portable_local_declarations(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let Some(portable) = step_render_context.portable_locals else {
return;
};
let Some(declarations) = portable.declarations.get(rule_index) else {
return;
};
for declaration in declarations {
writeln!(out, " #[allow(unused_mut)]\n {declaration}")
.expect("writing to a string cannot fail");
}
}
/// Declares the embedded per-rule attrs local (`__attrs`) on rule entry.
fn render_embedded_attrs_local(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let Some(embedded) = step_render_context.embedded else {
return;
};
if !embedded
.rule_has_attrs
.get(rule_index)
.copied()
.unwrap_or_default()
{
return;
}
let attrs_struct = embedded::attrs_struct_name(rule_index);
writeln!(
out,
" #[allow(unused_mut)]\n let mut __attrs = {attrs_struct}::default();"
)
.expect("writing to a string cannot fail");
if let Some(arg0) = embedded
.rule_arg0
.get(rule_index)
.and_then(Option::as_deref)
{
writeln!(
out,
" if let Some(__arg) = self.__embedded_pending_arg.take() {{ __attrs.{arg0} = __arg as _; }}"
)
.expect("writing to a string cannot fail");
}
}
/// Runs the embedded `@init` body at rule entry, after context/attrs setup and
/// before matching the rule body.
fn render_embedded_init_entry(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let Some(embedded) = step_render_context.embedded else {
return;
};
if let Some(init) = embedded.init_entry.get(&rule_index) {
writeln!(out, " {init}").expect("writing to a string cannot fail");
}
}
/// Runs the embedded `@after` body (committed path only — ANTLR's caught-error
/// path skips `@after`) and seals the attrs snapshot before `finish_rule`.
fn render_embedded_after_and_seal(
out: &mut String,
rule_index: usize,
step_render_context: GeneratedStepRenderContext<'_>,
run_after: bool,
) {
let Some(embedded) = step_render_context.embedded else {
return;
};
if run_after {
if let Some(after) = embedded.after.get(&rule_index) {
writeln!(out, " {after}").expect("writing to a string cannot fail");
}
}
if embedded
.rule_has_attrs
.get(rule_index)
.copied()
.unwrap_or_default()
{
writeln!(
out,
" __ctx.set_generated_attrs(antlr4_runtime::GeneratedAttrs::new(__attrs.clone()));"
)
.expect("writing to a string cannot fail");
}
}
fn render_generated_rule_method(
out: &mut String,
rule: &GeneratedParserRule,
step_render_context: GeneratedStepRenderContext<'_>,
) {
if rule.left_recursive {
render_generated_left_recursive_rule_method(out, rule, step_render_context);
return;
}
let index = rule.rule_index;
let entry_state = rule.entry_state;
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}(&mut self, __precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " let _ = __precedence;").expect("writing to a string cannot fail");
writeln!(out, " let _ = allow_fallback;").expect("writing to a string cannot fail");
writeln!(
out,
" let __generated_diagnostic_marker = self.base.generated_diagnostics_checkpoint();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __ctx = self.base.enter_rule({entry_state}isize, {index});"
)
.expect("writing to a string cannot fail");
// Capture the rule start AFTER `enter_rule`, which advances the cursor past any
// leading hidden-channel tokens to the first visible token. Capturing before
// would make `$start`/`$text` in generated actions include a leading hidden
// prefix (e.g. whitespace), diverging from ANTLR and the rule context start.
writeln!(
out,
" let __rule_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_portable_local_declarations(out, index, step_render_context);
render_embedded_attrs_local(out, index, step_render_context);
render_embedded_init_entry(out, index, step_render_context);
writeln!(out, " let mut __consumed_eof = false;")
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __sync_error: Option<antlr4_runtime::AntlrError> = None;"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let __result = (|| -> Result<(), antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
render_generated_steps(out, &rule.steps, 3, step_render_context);
writeln!(out, " Ok(())").expect("writing to a string cannot fail");
writeln!(out, " }})();").expect("writing to a string cannot fail");
writeln!(out, " match __result {{").expect("writing to a string cannot fail");
writeln!(out, " Ok(()) => {{").expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, true);
writeln!(
out,
" let __tree = self.base.finish_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " Err(__error) => {{").expect("writing to a string cannot fail");
// A rule's own `sync_decision` failure (`__sync_error`) is fatal ONLY at the
// top-level public entry (`allow_fallback`). When this rule is a nested child
// (`!allow_fallback`), ANTLR recovers the mismatch INSIDE the child and returns
// a partial subtree to the parent — it never propagates the sync failure up. So
// for a nested child, recover locally like any other body error (a `Fatal`
// escaping here would make the parent recover on ITS context, dropping the
// child subtree). Only the true top-level keeps the `Fatal` abort (preserving
// antlr#6 `InvalidEmptyInput`-style start-rule errors).
writeln!(
out,
" if let Some(__error) = __sync_error {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " if allow_fallback {{")
.expect("writing to a string cannot fail");
writeln!(out, " self.base.exit_rule();")
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.restore_generated_diagnostics(__generated_diagnostic_marker);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.record_generated_syntax_error();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" return Err(GeneratedRuleError::Fatal(__error));"
)
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " return Ok(__tree);")
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
}
fn render_generated_left_recursive_rule_method(
out: &mut String,
rule: &GeneratedParserRule,
step_render_context: GeneratedStepRenderContext<'_>,
) {
let index = rule.rule_index;
let entry_state = rule.entry_state;
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}(&mut self, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.parse_generated_rule_{index}_precedence(0, allow_fallback)"
)
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
"\n #[allow(dead_code)]\n fn parse_generated_rule_{index}_precedence(&mut self, __precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " let _ = allow_fallback;").expect("writing to a string cannot fail");
writeln!(
out,
" let __generated_diagnostic_marker = self.base.generated_diagnostics_checkpoint();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __ctx = self.base.enter_recursion_rule({entry_state}isize, {index}, __precedence);"
)
.expect("writing to a string cannot fail");
// Capture the rule start AFTER `enter_recursion_rule`, which (via `enter_rule`)
// advances the cursor past any leading hidden-channel tokens to the first
// visible token. Capturing before would make `$start`/`$text` in generated
// actions include a leading hidden prefix, diverging from ANTLR.
writeln!(
out,
" let __rule_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_portable_local_declarations(out, index, step_render_context);
render_embedded_attrs_local(out, index, step_render_context);
render_embedded_init_entry(out, index, step_render_context);
writeln!(out, " let mut __consumed_eof = false;")
.expect("writing to a string cannot fail");
writeln!(
out,
" let mut __sync_error: Option<antlr4_runtime::AntlrError> = None;"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" let __result = (|| -> Result<(), antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
render_generated_steps(out, &rule.steps, 3, step_render_context);
writeln!(out, " Ok(())").expect("writing to a string cannot fail");
writeln!(out, " }})();").expect("writing to a string cannot fail");
writeln!(out, " match __result {{").expect("writing to a string cannot fail");
writeln!(out, " Ok(()) => {{").expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, true);
writeln!(
out,
" let __tree = self.base.finish_recursion_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " Err(__error) => {{").expect("writing to a string cannot fail");
// Same as the non-left-recursive case: a nested child (`!allow_fallback`)
// recovers its own sync failure internally and returns a partial subtree; only
// the top-level entry propagates `Fatal`. Use `finish_recursion_rule` (which
// unrolls the recursion context) in the recover branch — do NOT also call
// `unroll_recursion_context` (that would double-unroll).
writeln!(
out,
" if let Some(__error) = __sync_error {{"
)
.expect("writing to a string cannot fail");
writeln!(out, " if allow_fallback {{")
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.unroll_recursion_context();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.restore_generated_diagnostics(__generated_diagnostic_marker);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" self.base.record_generated_syntax_error();"
)
.expect("writing to a string cannot fail");
writeln!(
out,
" return Err(GeneratedRuleError::Fatal(__error));"
)
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_recursion_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " return Ok(__tree);")
.expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(
out,
" self.base.recover_generated_rule(&mut __ctx, atn(), __error);"
)
.expect("writing to a string cannot fail");
render_embedded_after_and_seal(out, index, step_render_context, false);
writeln!(
out,
" let __tree = self.base.finish_recursion_rule(__ctx, __consumed_eof);"
)
.expect("writing to a string cannot fail");
writeln!(out, " Ok(__tree)").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
writeln!(out, " }}").expect("writing to a string cannot fail");
}
fn render_generated_steps(
out: &mut String,
steps: &[GeneratedParserStep],
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
for step in steps {
render_generated_step(out, step, indent, render_context);
}
}
fn render_generated_step(
out: &mut String,
step: &GeneratedParserStep,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let pad = " ".repeat(indent);
match step {
GeneratedParserStep::MatchToken {
token_type,
follow_state,
} => {
writeln!(
out,
"{pad}let __match = self.base.match_token_recovering({token_type}, {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::MatchSet {
token_set,
intervals,
follow_state,
} => {
if let Some(token_set) = token_set {
writeln!(
out,
"{pad}let __match = self.base.match_token_set_recovering(atn().token_set({token_set}).expect(\"generated parser token-set index\"), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
} else {
let intervals = render_i32_ranges(intervals);
writeln!(
out,
"{pad}let __match = self.base.match_set_recovering(&{intervals}, {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
}
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::MatchNotSet {
token_set,
intervals,
follow_state,
} => {
if let Some(token_set) = token_set {
writeln!(
out,
"{pad}let __match = self.base.match_not_token_set_recovering(atn().token_set({token_set}).expect(\"generated parser token-set index\"), 1, atn().max_token_type(), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
} else {
let intervals = render_i32_ranges(intervals);
writeln!(
out,
"{pad}let __match = self.base.match_not_set_recovering(&{intervals}, 1, atn().max_token_type(), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
}
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::MatchWildcard { follow_state } => {
// A wildcard matches any single token. Model it as a not-set with an
// empty exclusion set (every token in 1..=max), reusing the recovering
// match so a wildcard at EOF performs ANTLR's single-token insertion
// (`<missing ...>` error node) and lets the rule continue, instead of
// aborting the remaining steps.
writeln!(
out,
"{pad}let __match = self.base.match_not_set_recovering(&[], 1, atn().max_token_type(), {follow_state}, atn())?;"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}__consumed_eof |= __match.consumed_eof();")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}for __child in __match.into_child_iter() {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
}
GeneratedParserStep::Precedence(precedence) => {
writeln!(out, "{pad}if !self.base.precpred({precedence}) {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"precpred(_ctx, {precedence})\"));"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
GeneratedParserStep::Predicate {
rule_index,
pred_index,
} => {
if let Some((condition, message)) = render_context
.portable_locals
.and_then(|portable| portable.predicates.get(&(*rule_index, *pred_index)))
{
writeln!(out, "{pad}if !({condition}) {{")
.expect("writing to a string cannot fail");
match message {
Some(message) => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_option_error({rule_index}, \"{}\".to_owned()));",
rust_string(message)
)
.expect("writing to a string cannot fail"),
None => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"semantic predicate\"));"
)
.expect("writing to a string cannot fail"),
}
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
return;
}
if let Some(embedded) = render_context.embedded {
let (condition, message) =
embedded_predicate_condition_and_message(&embedded, *rule_index, *pred_index);
writeln!(out, "{pad}if !({condition}) {{")
.expect("writing to a string cannot fail");
match message {
Some(message) => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_option_error({rule_index}, \"{}\".to_owned()));",
rust_string(&message)
)
.expect("writing to a string cannot fail"),
None => writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"semantic predicate\"));"
)
.expect("writing to a string cannot fail"),
}
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
return;
}
writeln!(
out,
"{pad}if !self.base.parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), {rule_index}, {pred_index}, &__ctx, __precedence) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} if let Some(__message) = self.base.parser_semantic_ir_predicate_failure_message({rule_index}, {pred_index}, parser_semantics()) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} return Err(self.base.failed_predicate_option_error({rule_index}, __message));"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} return Err(self.base.failed_predicate_error(\"semantic predicate\"));"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
GeneratedParserStep::CallRule {
source_state,
rule_index,
precedence,
} => {
writeln!(
out,
"{pad}let __invoking_marker = self.base.push_invoking_state({source_state}isize);"
)
.expect("writing to a string cannot fail");
if let Some(embedded) = render_context.embedded {
if let Some(expression) = embedded.call_args.get(source_state) {
writeln!(
out,
"{pad}self.__embedded_pending_arg = Some(i64::from({expression}));"
)
.expect("writing to a string cannot fail");
}
}
let precedence = match precedence {
GeneratedRuleCallPrecedence::Literal(value) => value.to_string(),
GeneratedRuleCallPrecedence::InheritLocal => "__precedence".to_owned(),
};
let from_generated_call =
format!("self.parse_rule_precedence_from_generated({rule_index}, {precedence})");
let generated_child_call = if render_context
.direct_generated_rule_calls
.get(*rule_index)
.copied()
.unwrap_or_default()
{
format!(
"self.parse_generated_rule_{rule_index}_dispatch({precedence}, false).map_err(GeneratedRuleError::into_error)"
)
} else {
from_generated_call.clone()
};
let child_call = if render_context
.atn_preferred_rule_calls
.get(*rule_index)
.copied()
.unwrap_or_default()
{
// ATN-preferred child: route through `parse_rule_precedence_from_generated`.
// The rule's `parse_generated_rule` dispatch arm is guarded by
// `generated_only()`, so in normal mode the generated probe returns
// `None` and the wrapper parses the child on the INTERPRETED path
// (preserving the ATN-preferred optimization).
from_generated_call
} else {
generated_child_call
};
writeln!(out, "{pad}let __child = {child_call};")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad}self.base.discard_invoking_state(__invoking_marker);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}let __child = __child?;").expect("writing to a string cannot fail");
writeln!(out, "{pad}self.base.add_parse_child(&mut __ctx, __child);")
.expect("writing to a string cannot fail");
}
GeneratedParserStep::Action {
source_state,
rule_index,
} => {
writeln!(
out,
"{pad}let action = self.base.parser_action_at_current({source_state}, {rule_index}, __rule_start, __consumed_eof);"
)
.expect("writing to a string cannot fail");
if let Some(statement) = render_context.inline_action_statements.get(source_state) {
if !statement.is_empty() {
writeln!(out, "{pad}{statement}").expect("writing to a string cannot fail");
}
}
if render_context.embedded.is_some() {
// Embedded actions executed inline just above, at their
// ANTLR-correct point in the rule body.
writeln!(out, "{pad}let _ = &action;").expect("writing to a string cannot fail");
return;
}
writeln!(out, "{pad}let _ = action;").expect("writing to a string cannot fail");
}
GeneratedParserStep::Decision {
state,
decision,
track_alt_number,
allow_semantic_context,
force_context,
fast_path,
alts,
} => {
render_generated_decision(
out,
DecisionRender {
state: *state,
decision: *decision,
track_alt_number: *track_alt_number,
allow_semantic_context: *allow_semantic_context,
force_context: *force_context,
fast_path: fast_path.as_ref(),
alts,
},
indent,
render_context,
);
}
GeneratedParserStep::StarLoop {
state,
decision,
enter_alt,
exit_alt,
track_alt_number,
allow_semantic_context,
force_context,
plus_loop,
fast_path,
body,
} => {
render_generated_star_loop(
out,
StarLoopRender {
state: *state,
decision: *decision,
alts: (*enter_alt, *exit_alt),
track_alt_number: *track_alt_number,
allow_semantic_context: *allow_semantic_context,
force_context: *force_context,
plus_loop: *plus_loop,
fast_path: fast_path.as_ref(),
body,
},
indent,
render_context,
);
}
GeneratedParserStep::LeftRecursiveLoop {
state,
decision,
enter_alt,
exit_alt,
rule_index,
entry_state,
body,
..
} => {
render_generated_left_recursive_loop(
out,
LeftRecursiveLoopRender {
state: *state,
decision: *decision,
alts: (*enter_alt, *exit_alt),
rule: (*rule_index, *entry_state),
body,
},
indent,
render_context,
);
}
}
}
fn render_generated_decision(
out: &mut String,
decision_info: DecisionRender<'_>,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let DecisionRender {
state,
decision,
track_alt_number,
allow_semantic_context,
force_context,
fast_path,
alts,
} = decision_info;
let pad = " ".repeat(indent);
// A tool-LL(1) decision dispatches on the tool's complete LOOK table
// (exit alternatives included), like Java's switch compilation.
let tool_fast_path = render_context
.embedded
.and_then(|embedded| embedded.tool_ll1_fast_path(decision));
let fast_path = tool_fast_path.as_ref().or(fast_path);
if let Some(fast_path) = fast_path.filter(|_| {
!allow_semantic_context
&& !force_context
&& !render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision))
}) {
writeln!(
out,
"{pad}let mut __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}let __prediction = match self.base.la(1) {{")
.expect("writing to a string cannot fail");
render_generated_fast_prediction_arms(out, &pad, fast_path);
writeln!(out, "{pad} _ => {{").expect("writing to a string cannot fail");
// A non-loop block/optional decision is never a loop-back: ANTLR syncs it
// like BLOCK_START (single-token deletion), so pass `false`.
render_generated_sync_decision(out, &format!("{pad} "), state, "false");
writeln!(
out,
"{pad} __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_generated_ll1_then_adaptive_prediction(
out,
&format!("{pad} "),
state,
decision,
false,
);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
} else {
if !allow_semantic_context {
render_generated_sync_decision(out, &pad, state, "false");
}
writeln!(
out,
"{pad}let __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
let force_adaptive = render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision));
if allow_semantic_context || force_context {
render_generated_adaptive_prediction(out, &pad, decision);
} else if force_adaptive {
render_generated_two_stage_adaptive_assignment(out, &pad, decision);
} else {
render_generated_ll1_then_adaptive_prediction(out, &pad, state, decision, true);
}
}
if allow_semantic_context {
render_generated_semantic_prediction_filter(
out,
&pad,
alts,
render_context.embedded,
render_context.portable_locals,
);
render_generated_decision_diagnostic_report(
out,
&pad,
state,
alts,
render_context.embedded,
render_context.portable_locals,
);
} else {
writeln!(
out,
"{pad}self.base.record_generated_prediction_diagnostic(atn(), {state}, &__prediction);"
)
.expect("writing to a string cannot fail");
}
writeln!(out, "{pad}match __prediction.alt {{").expect("writing to a string cannot fail");
for (index, steps) in alts.iter().enumerate() {
let alt = index + 1;
writeln!(out, "{pad} {alt} => {{").expect("writing to a string cannot fail");
render_generated_alt_number_assignment(
out,
&format!("{pad} "),
alt,
render_context.track_alt_numbers && track_alt_number,
);
render_generated_steps(out, steps, indent + 2, render_context);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(
out,
"{pad} _ => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_fast_prediction_arms(
out: &mut String,
pad: &str,
fast_path: &GeneratedDecisionFastPath,
) {
for arm in &fast_path.arms {
let patterns = render_i32_match_patterns(&arm.intervals);
let alt = arm.alt;
writeln!(
out,
"{pad} {patterns} => antlr4_runtime::ParserAtnPrediction {{ alt: {alt}, requires_full_context: false, has_semantic_context: false, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
}
}
/// Two-stage adaptive prediction without the LL(1) shortcut — Java's plain
/// `adaptivePredict`: the SLL probe resolves or flags a full-context
/// conflict, and the retry with real outer context only runs when the
/// parser's prediction mode allows it (never in SLL mode).
fn render_generated_two_stage_adaptive_assignment(out: &mut String, pad: &str, decision: usize) {
writeln!(out, "{pad}let __prediction = {{").expect("writing to a string cannot fail");
render_generated_sll_then_context_prediction_with_indent(out, pad, decision, 1);
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn render_generated_ll1_then_adaptive_prediction(
out: &mut String,
pad: &str,
state: usize,
decision: usize,
assign: bool,
) {
let prefix = if assign { "let __prediction = " } else { "" };
let suffix = if assign { ";" } else { "" };
writeln!(
out,
"{pad}{prefix}if let Some(__prediction) = self.base.ll1_decision_prediction(atn(), {state}) {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad}}} else {{").expect("writing to a string cannot fail");
render_generated_sll_then_context_prediction_with_indent(out, pad, decision, 1);
writeln!(out, "{pad}}}{suffix}").expect("writing to a string cannot fail");
}
fn render_generated_decision_diagnostic_report(
out: &mut String,
pad: &str,
state: usize,
alts: &[Vec<GeneratedParserStep>],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) {
let alt_conditions = alts
.iter()
.map(|steps| {
semantic_alt_candidate_condition_with_la(steps, "__diagnostic_la", embedded, portable)
})
.collect::<Vec<_>>();
if alt_conditions
.iter()
.any(|condition| condition == "true" || condition == "false")
{
return;
}
writeln!(out, "{pad}if self.base.report_diagnostic_errors() {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __diagnostic_la = self.base.la(1);")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let mut __diagnostic_alts = Vec::new();")
.expect("writing to a string cannot fail");
for (index, condition) in alt_conditions.iter().enumerate() {
let alt = index + 1;
writeln!(out, "{pad} if {condition} {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __diagnostic_alts.push({alt});")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(
out,
"{pad} self.base.record_generated_ambiguity_diagnostic(atn(), {state}, __decision_start, __decision_start, &__diagnostic_alts);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_semantic_prediction_filter(
out: &mut String,
pad: &str,
alts: &[Vec<GeneratedParserStep>],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) {
let alt_has_predicates = alts
.iter()
.map(|steps| !leading_predicates(steps, portable).is_empty())
.collect::<Vec<_>>();
if !alt_has_predicates
.iter()
.any(|has_predicate| *has_predicate)
{
return;
}
let alt_conditions = alts
.iter()
.map(|steps| semantic_alt_candidate_condition(steps, embedded, portable))
.collect::<Vec<_>>();
writeln!(
out,
"{pad}let __prediction = if __prediction.has_semantic_context {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __semantic_la = self.base.la(1);")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __semantic_alt = match __prediction.alt {{"
)
.expect("writing to a string cannot fail");
for (index, condition) in alt_conditions.iter().enumerate() {
if !alt_has_predicates[index] {
continue;
}
let alt = index + 1;
writeln!(out, "{pad} {alt} if {condition} => Some({alt}),")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} {alt} => {{").expect("writing to a string cannot fail");
render_semantic_alt_search(out, pad, &alt_conditions, alts, portable);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(out, "{pad} _ => Some(__prediction.alt),")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }};").expect("writing to a string cannot fail");
writeln!(out, "{pad} match __semantic_alt {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Some(__alt) => antlr4_runtime::ParserAtnPrediction {{ alt: __alt, ..__prediction }},"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} None => {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __error = self.base.no_viable_alternative_error(__decision_start);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} return Err(__error);")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}} else {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn render_semantic_alt_search(
out: &mut String,
pad: &str,
alt_conditions: &[String],
alts: &[Vec<GeneratedParserStep>],
portable: Option<PortableLocalStepRender<'_>>,
) {
// The predicted alt's predicate failed; pick another alt whose candidate
// condition holds. This runs in TWO passes so an alt whose viability is not
// locally checkable (no predicate and no computable lookahead — its first
// consuming step is a rule call/decision/loop, giving condition `"true"`)
// neither shadows a concretely-guarded alt nor becomes unreachable:
//
// Pass 1 — alts with a resolved guard (predicate and/or lookahead), in
// order. A later token-led alt is reachable even if an earlier
// unresolved rule-call alt exists (`{p()}? 'a' | x | 'a'` on input `a`
// picks the `'a'` alt, not rule-call `x`).
// Pass 2 — the remaining unresolved alts, in order, as a last resort. So
// an unresolved alt is still tried when no resolved alt matched
// (`{p()}? 'a' | x` on input in FIRST(x) selects `x` instead of
// reporting NoViableAlt).
//
// Scoped to the fallback search only; the shared
// `semantic_alt_candidate_condition` is unchanged, so left-recursion
// loop-entry and diagnostic paths keep their behavior.
let unresolved = alts
.iter()
.map(|steps| semantic_alt_guard_is_unresolved(steps, portable))
.collect::<Vec<_>>();
for (index, condition) in alt_conditions.iter().enumerate() {
if unresolved.get(index).copied().unwrap_or(false) {
continue;
}
let alt = index + 1;
writeln!(out, "{pad} if {condition} {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} Some({alt})").expect("writing to a string cannot fail");
writeln!(out, "{pad} }} else").expect("writing to a string cannot fail");
}
// Last-resort pass: unresolved alts keep their real condition (typically
// `true`), so they are tried only after every resolved alt missed.
for (index, condition) in alt_conditions.iter().enumerate() {
if !unresolved.get(index).copied().unwrap_or(false) {
continue;
}
let alt = index + 1;
writeln!(out, "{pad} if {condition} {{")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} Some({alt})").expect("writing to a string cannot fail");
writeln!(out, "{pad} }} else").expect("writing to a string cannot fail");
}
writeln!(out, "{pad} {{ None }}").expect("writing to a string cannot fail");
}
/// Looks up the verbatim predicate expression (and optional `<fail=…>`
/// message) for a coordinate in embedded mode. A coordinate with no embedded
/// body evaluates true, matching ANTLR's treatment of a missing predicate.
fn embedded_predicate_condition_and_message(
embedded: &EmbeddedStepRender<'_>,
rule_index: usize,
pred_index: usize,
) -> (String, Option<String>) {
embedded
.predicates
.get(&(rule_index, pred_index))
.map_or_else(
|| ("true".to_owned(), None),
|(expression, message)| (expression.clone(), message.clone()),
)
}
fn semantic_alt_candidate_condition(
steps: &[GeneratedParserStep],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) -> String {
semantic_alt_candidate_condition_with_la(steps, "__semantic_la", embedded, portable)
}
fn semantic_alt_candidate_condition_with_la(
steps: &[GeneratedParserStep],
la_symbol: &str,
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) -> String {
// Order matters: the lookahead guard comes FIRST so `&&` short-circuits on it
// before any predicate hook runs. Otherwise, searching alternatives in a
// semantic decision would evaluate an alternative's leading hook/unknown
// predicate even when its first token cannot match the current lookahead —
// recording a spurious fail-loud `Unsupported` hit under
// `--sem-unknown=hook`/`error` and rejecting a later syntactically viable
// alternative. This also matches ANTLR, which only evaluates a predicate for
// a lookahead-viable alternative.
let mut conditions = Vec::new();
if let Some(lookahead) = leading_lookahead_condition(steps, la_symbol) {
conditions.push(lookahead);
}
conditions.extend(leading_predicates(steps, portable).into_iter().map(
|(rule_index, pred_index)| {
if let Some((condition, _)) =
portable.and_then(|portable| portable.predicates.get(&(rule_index, pred_index)))
{
return format!("({condition})");
}
embedded.map_or_else(
|| {
format!(
"self.base.parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), {rule_index}, {pred_index}, &__ctx, __precedence)"
)
},
|embedded| {
let (condition, _) = embedded_predicate_condition_and_message(
&embedded, rule_index, pred_index,
);
format!("({condition})")
},
)
},
));
if conditions.is_empty() {
"true".to_owned()
} else {
conditions.join(" && ")
}
}
/// Whether an alternative has no locally-checkable viability guard: no leading
/// predicate and no computable lookahead (its first consuming step is a
/// `CallRule` / nested decision / loop whose FIRST set is not computed here).
/// Such an alt's [`semantic_alt_candidate_condition`] is `"true"`, so in the
/// ordered semantic-alt fallback search it would shadow a later alt with a
/// concrete matching lookahead. The search treats these as last-resort
/// candidates instead. A genuine epsilon alt (no consuming step at all) is NOT
/// unguarded in this sense — it legitimately matches anything.
fn semantic_alt_guard_is_unresolved(
steps: &[GeneratedParserStep],
portable: Option<PortableLocalStepRender<'_>>,
) -> bool {
if !leading_predicates(steps, portable).is_empty() {
return false;
}
if leading_lookahead_condition(steps, "__semantic_la").is_some() {
return false;
}
// No predicate and no computable lookahead: unresolved only if a consuming
// step exists (rule call / decision / loop). Pure epsilon stays resolved.
steps.iter().any(|step| {
matches!(
step,
GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::Decision { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. }
)
})
}
fn leading_predicates(
steps: &[GeneratedParserStep],
portable: Option<PortableLocalStepRender<'_>>,
) -> Vec<(usize, usize)> {
let mut predicates = Vec::new();
for step in steps {
match step {
GeneratedParserStep::Predicate {
rule_index,
pred_index,
} => predicates.push((*rule_index, *pred_index)),
// Portable assignments run only after the alternative is selected,
// so predicates after one are not prediction-visible.
GeneratedParserStep::Action { source_state, .. }
if portable
.is_some_and(|portable| portable.inline_actions.contains_key(source_state)) =>
{
break;
}
GeneratedParserStep::Action { .. } | GeneratedParserStep::Precedence(_) => {}
GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::Decision { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => break,
}
}
predicates
}
fn leading_lookahead_condition(steps: &[GeneratedParserStep], la_symbol: &str) -> Option<String> {
for step in steps {
match step {
GeneratedParserStep::Predicate { .. }
| GeneratedParserStep::Action { .. }
| GeneratedParserStep::Precedence(_) => {}
GeneratedParserStep::MatchToken { token_type, .. } => {
return Some(format!("{la_symbol} == {token_type}"));
}
GeneratedParserStep::MatchSet {
token_set,
intervals,
..
} => {
return Some(token_set_condition(la_symbol, *token_set, intervals));
}
GeneratedParserStep::MatchNotSet {
token_set,
intervals,
..
} => {
let excluded = token_set_condition(la_symbol, *token_set, intervals);
return Some(format!(
"(1..=atn().max_token_type()).contains(&{la_symbol}) && !({excluded})"
));
}
GeneratedParserStep::MatchWildcard { .. } => {
return Some(format!("{la_symbol} != antlr4_runtime::TOKEN_EOF"));
}
GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::Decision { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => return None,
}
}
None
}
fn token_set_condition(symbol: &str, token_set: Option<usize>, intervals: &[(i32, i32)]) -> String {
token_set.map_or_else(
|| intervals_condition(symbol, intervals),
|token_set| {
format!(
"atn().token_set({token_set}).expect(\"generated parser token-set index\").contains({symbol})"
)
},
)
}
fn intervals_condition(symbol: &str, intervals: &[(i32, i32)]) -> String {
if intervals.is_empty() {
return "false".to_owned();
}
intervals
.iter()
.map(|(start, stop)| {
if start == stop {
format!("{symbol} == {start}")
} else {
format!("({start}..={stop}).contains(&{symbol})")
}
})
.collect::<Vec<_>>()
.join(" || ")
}
fn render_generated_alt_number_assignment(out: &mut String, pad: &str, alt: usize, enabled: bool) {
if !enabled {
return;
}
writeln!(out, "{pad}if __ctx.alt_number() == 0 {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __ctx.set_alt_number({alt});")
.expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_sync_decision(out: &mut String, pad: &str, state: usize, loop_back_expr: &str) {
writeln!(
out,
"{pad}match self.base.sync_decision(atn(), {state}, !__ctx.has_matched_child(), {loop_back_expr}) {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} Ok(__sync_children) => {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} for __child in __sync_children {{ self.base.add_parse_child(&mut __ctx, __child); }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} Err(__error) => {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __sync_error = Some(__error.clone());")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} return Err(__error);").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_adaptive_prediction(out: &mut String, pad: &str, decision: usize) {
writeln!(out, "{pad}let __prediction = {{").expect("writing to a string cannot fail");
render_generated_adaptive_prediction_with_indent(out, pad, decision, 1);
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn render_generated_adaptive_prediction_with_indent(
out: &mut String,
pad: &str,
decision: usize,
extra_indent: usize,
) {
let nested = format!("{pad}{}", " ".repeat(extra_indent));
writeln!(
out,
"{nested}let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested}let __prediction_context = __simulator.intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested}__simulator.set_exact_ambig_detection(self.base.prediction_mode() == antlr4_runtime::PredictionMode::LlExactAmbigDetection);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested}__simulator.adaptive_predict_stream_info_with_context({decision}, 0, self.base.input(), __prediction_context)"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} .map_err(|__error| match __error {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ index, .. }} => self.base.no_viable_alternative_error_at(__decision_start, index),"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} _ => self.base.no_viable_alternative_error(__decision_start),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} }})?").expect("writing to a string cannot fail");
}
fn render_generated_sll_then_context_prediction_with_indent(
out: &mut String,
pad: &str,
decision: usize,
extra_indent: usize,
) {
let nested = format!("{pad}{}", " ".repeat(extra_indent));
writeln!(out, "{nested}let __prediction = {{").expect("writing to a string cannot fail");
writeln!(
out,
"{nested} let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
// Stage 1 uses the SLL probe: on a full-context-requiring conflict it returns
// requires_full_context WITHOUT running the LL loop (the result is discarded
// here anyway — only the boolean gates the stage-2 re-run with real context).
writeln!(
out,
"{nested} __simulator.adaptive_predict_stream_info_sll_probe({decision}, 0, self.base.input())"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} .map_err(|__error| match __error {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ index, .. }} => self.base.no_viable_alternative_error_at(__decision_start, index),"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{nested} _ => self.base.no_viable_alternative_error(__decision_start),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{nested} }})?").expect("writing to a string cannot fail");
writeln!(out, "{nested}}};").expect("writing to a string cannot fail");
writeln!(
out,
"{nested}if __prediction.requires_full_context && self.base.prediction_mode() != antlr4_runtime::PredictionMode::Sll {{"
)
.expect("writing to a string cannot fail");
render_generated_adaptive_prediction_with_indent(out, pad, decision, extra_indent + 1);
writeln!(out, "{nested}}} else {{").expect("writing to a string cannot fail");
writeln!(out, "{nested} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{nested}}}").expect("writing to a string cannot fail");
}
fn render_generated_star_loop(
out: &mut String,
loop_info: StarLoopRender<'_>,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let StarLoopRender {
state,
decision,
alts,
track_alt_number,
allow_semantic_context,
force_context,
plus_loop,
fast_path,
body,
} = loop_info;
let (enter_alt, exit_alt) = alts;
let pad = " ".repeat(indent);
// A tool-LL(1) loop decision dispatches on the tool's complete LOOK
// table (exit alternative included), like Java's switch-driven loops.
let tool_fast_path = render_context
.embedded
.and_then(|embedded| embedded.tool_ll1_fast_path(decision));
let fast_path = tool_fast_path.as_ref().or(fast_path);
// Per-loop "iteration started" flag, threaded into `sync_decision` so it
// recovers like ANTLR: a `*` loop's first sync is at the loop ENTRY
// (single-token deletion), every later sync is a loop-BACK (multi-token
// `consumeUntil`). A `+` loop's mandatory first element is iteration 1, so it
// is already on the loop-back side at its first sync (init `true`).
let loop_iter = format!("__loop_iter_{state}");
writeln!(out, "{pad}let mut {loop_iter} = {plus_loop};")
.expect("writing to a string cannot fail");
writeln!(out, "{pad}loop {{").expect("writing to a string cannot fail");
let inner_pad = format!("{pad} ");
if let Some(fast_path) = fast_path.filter(|_| {
!allow_semantic_context
&& !force_context
&& !render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision))
}) {
writeln!(
out,
"{pad} let mut __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __prediction = match self.base.la(1) {{")
.expect("writing to a string cannot fail");
render_generated_fast_prediction_arms(out, &inner_pad, fast_path);
writeln!(out, "{pad} _ => {{").expect("writing to a string cannot fail");
render_generated_sync_decision(out, &format!("{pad} "), state, &loop_iter);
writeln!(
out,
"{pad} __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
render_generated_ll1_then_adaptive_prediction(
out,
&format!("{pad} "),
state,
decision,
false,
);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} }};").expect("writing to a string cannot fail");
} else {
render_generated_sync_decision(out, &inner_pad, state, &loop_iter);
writeln!(
out,
"{pad} let __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
let force_adaptive = render_context
.embedded
.is_some_and(|embedded| embedded.adaptive_decision(decision));
if allow_semantic_context || force_context {
render_generated_adaptive_prediction(out, &inner_pad, decision);
} else if force_adaptive {
render_generated_two_stage_adaptive_assignment(out, &inner_pad, decision);
} else {
render_generated_ll1_then_adaptive_prediction(out, &inner_pad, state, decision, true);
}
}
render_generated_loop_semantic_prediction_filter(
out,
&format!("{pad} "),
enter_alt,
exit_alt,
body,
render_context.embedded,
render_context.portable_locals,
);
writeln!(
out,
"{pad} self.base.record_generated_prediction_diagnostic(atn(), {state}, &__prediction);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} match __prediction.alt {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} {enter_alt} => {{").expect("writing to a string cannot fail");
// Once an iteration is taken, every subsequent sync is a loop-back.
writeln!(out, "{pad} {loop_iter} = true;").expect("writing to a string cannot fail");
render_generated_alt_number_assignment(
out,
&format!("{pad} "),
enter_alt,
render_context.track_alt_numbers && track_alt_number,
);
render_generated_steps(out, body, indent + 3, render_context);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} {exit_alt} => {{").expect("writing to a string cannot fail");
render_generated_alt_number_assignment(
out,
&format!("{pad} "),
exit_alt,
render_context.track_alt_numbers && track_alt_number,
);
writeln!(out, "{pad} break;").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} _ => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
fn render_generated_left_recursive_loop(
out: &mut String,
loop_info: LeftRecursiveLoopRender<'_>,
indent: usize,
render_context: GeneratedStepRenderContext<'_>,
) {
let LeftRecursiveLoopRender {
state,
decision,
alts,
rule,
body,
} = loop_info;
let (rule_index, entry_state) = rule;
let (enter_alt, exit_alt) = alts;
let pad = " ".repeat(indent);
writeln!(out, "{pad}loop {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __decision_start = antlr4_runtime::IntStream::index(self.base.input());"
)
.expect("writing to a string cannot fail");
if render_context.embedded.is_some() {
writeln!(
out,
"{pad} let __prediction_precedence = if __precedence <= 0 {{ 0 }} else {{ __precedence as usize }};"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __prediction = match {{")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __prediction_context = __simulator.intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} __simulator.set_exact_ambig_detection(self.base.prediction_mode() == antlr4_runtime::PredictionMode::LlExactAmbigDetection);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} __simulator.adaptive_predict_stream_info_with_context({decision}, __prediction_precedence, self.base.input(), __prediction_context)"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }} {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} Ok(__prediction) => __prediction,")
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ .. }}) if self.base.left_recursive_loop_enter_matches(atn(), {state}, __precedence) => {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} antlr4_runtime::ParserAtnPrediction {{ alt: {enter_alt}, requires_full_context: true, has_semantic_context: true, diagnostic: None }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ .. }}) => {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, requires_full_context: true, has_semantic_context: false, diagnostic: None }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(_) => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
} else {
writeln!(
out,
"{pad} let __prediction = match self.base.left_recursive_loop_enter_prediction(atn(), {state}, __precedence) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Some(true) => antlr4_runtime::ParserAtnPrediction {{ alt: {enter_alt}, requires_full_context: false, has_semantic_context: true, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Some(false) => antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, requires_full_context: false, has_semantic_context: false, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} None => {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __prediction_precedence = if __precedence <= 0 {{ 0 }} else {{ __precedence as usize }};"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __simulator = self.simulator.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} let __prediction_context = __simulator.intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()));"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} __simulator.set_exact_ambig_detection(self.base.prediction_mode() == antlr4_runtime::PredictionMode::LlExactAmbigDetection);"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} match __simulator.adaptive_predict_stream_info_with_context({decision}, __prediction_precedence, self.base.input(), __prediction_context) {{"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Ok(__prediction) => __prediction,"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt {{ .. }}) => antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, requires_full_context: true, has_semantic_context: false, diagnostic: None }},"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"{pad} Err(_) => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
}
writeln!(out, "{pad} }};").expect("writing to a string cannot fail");
render_generated_loop_semantic_prediction_filter(
out,
&format!("{pad} "),
enter_alt,
exit_alt,
body,
render_context.embedded,
render_context.portable_locals,
);
writeln!(
out,
"{pad} self.base.record_generated_prediction_diagnostic(atn(), {state}, &__prediction);"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} match __prediction.alt {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} {enter_alt} => {{").expect("writing to a string cannot fail");
if render_context.embedded.is_some_and(|embedded| {
embedded
.rule_has_attrs
.get(rule_index)
.copied()
.unwrap_or(false)
}) {
// Java's `_prevctx`: the outgoing iteration context becomes the new
// context's left child, so it must carry the attrs accumulated so
// far — operator-alt actions and typed views read them through the
// child context, not the live `__attrs` local.
writeln!(
out,
"{pad} __ctx.set_generated_attrs(antlr4_runtime::GeneratedAttrs::new(__attrs.clone()));"
)
.expect("writing to a string cannot fail");
}
writeln!(
out,
"{pad} self.base.push_new_recursion_context_with_previous({entry_state}isize, {rule_index}, &mut __ctx);"
)
.expect("writing to a string cannot fail");
render_generated_steps(out, body, indent + 3, render_context);
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad} {exit_alt} => break,").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} _ => return Err(self.base.no_viable_alternative_error(__decision_start)),"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}}").expect("writing to a string cannot fail");
}
#[allow(clippy::too_many_arguments)]
fn render_generated_loop_semantic_prediction_filter(
out: &mut String,
pad: &str,
enter_alt: usize,
exit_alt: usize,
body: &[GeneratedParserStep],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) {
let Some(condition) = loop_entry_condition(body, embedded, portable) else {
return;
};
writeln!(
out,
"{pad}let __prediction = if __prediction.alt == {enter_alt} {{"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} let __semantic_la = self.base.la(1);")
.expect("writing to a string cannot fail");
writeln!(out, "{pad} if {condition} {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad} }} else {{").expect("writing to a string cannot fail");
writeln!(
out,
"{pad} antlr4_runtime::ParserAtnPrediction {{ alt: {exit_alt}, ..__prediction }}"
)
.expect("writing to a string cannot fail");
writeln!(out, "{pad} }}").expect("writing to a string cannot fail");
writeln!(out, "{pad}}} else {{").expect("writing to a string cannot fail");
writeln!(out, "{pad} __prediction").expect("writing to a string cannot fail");
writeln!(out, "{pad}}};").expect("writing to a string cannot fail");
}
fn loop_entry_condition(
body: &[GeneratedParserStep],
embedded: Option<EmbeddedStepRender<'_>>,
portable: Option<PortableLocalStepRender<'_>>,
) -> Option<String> {
let step = body.first()?;
match step {
GeneratedParserStep::Predicate { .. } | GeneratedParserStep::Precedence(_) => {
Some(semantic_alt_candidate_condition(body, embedded, portable))
}
GeneratedParserStep::Decision { alts, .. } => {
if !alts.iter().any(|alt| steps_contain_predicate(alt)) {
return None;
}
Some(
alts.iter()
.map(|alt| {
format!(
"({})",
semantic_alt_candidate_condition(alt, embedded, portable)
)
})
.collect::<Vec<_>>()
.join(" || "),
)
}
GeneratedParserStep::Action { .. }
| GeneratedParserStep::MatchToken { .. }
| GeneratedParserStep::MatchSet { .. }
| GeneratedParserStep::MatchNotSet { .. }
| GeneratedParserStep::MatchWildcard { .. }
| GeneratedParserStep::CallRule { .. }
| GeneratedParserStep::StarLoop { .. }
| GeneratedParserStep::LeftRecursiveLoop { .. } => None,
}
}
#[allow(clippy::fn_params_excessive_bools)]
fn render_parser_parse_rule_fallback(
track_alt_numbers: bool,
_predicates: &[((usize, usize), PredicateTemplate)],
rule_args: &[(usize, usize, RuleArgTemplate)],
has_action_dispatch: bool,
has_predicate_dispatch: bool,
unknown_policy_literal: Option<&str>,
) -> String {
let mut out = String::new();
if has_predicate_dispatch || unknown_policy_literal.is_some() {
writeln!(
out,
"let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ track_alt_numbers: {track_alt_numbers}, predicates: &[], semantics: Some(parser_semantics()), rule_args: &{}, member_actions: &[], return_actions: &[], unknown_predicate_policy: {} , ..antlr4_runtime::ParserRuntimeOptions::default() }})?;",
render_parser_rule_arg_array(rule_args),
unknown_policy_literal
.unwrap_or("antlr4_runtime::UnknownSemanticPolicy::AssumeTrue")
)
.expect("writing to a string cannot fail");
} else if track_alt_numbers {
writeln!(
out,
"let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ track_alt_numbers: true, ..antlr4_runtime::ParserRuntimeOptions::default() }})?;"
)
.expect("writing to a string cannot fail");
} else if has_action_dispatch {
writeln!(
out,
"let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions::default())?;"
)
.expect("writing to a string cannot fail");
} else {
return "self.base.parse_atn_rule_with_precedence(atn(), rule_index, precedence)"
.to_owned();
}
if has_action_dispatch {
writeln!(
out,
"for action in actions {{ self.run_action(action, tree); }}"
)
.expect("writing to a string cannot fail");
} else {
writeln!(out, "let _ = actions;").expect("writing to a string cannot fail");
}
writeln!(out, "Ok(tree)").expect("writing to a string cannot fail");
out.lines()
.map(|line| format!(" {line}"))
.collect::<Vec<_>>()
.join("\n")
}
/// Renders a Rust parser module with one public method per grammar rule.
///
/// Parser methods use generated recursive-descent bodies for the ATN subset
/// covered by `parser_generated_rules` and keep the interpreter fallback for
/// unsupported constructs while the generated surface is expanded.
#[cfg(test)]
fn render_parser(
grammar_name: &str,
data: &InterpData,
grammar_source: Option<&str>,
) -> io::Result<String> {
render_parser_with_options(
grammar_name,
data,
grammar_source,
ParserRenderOptions::default(),
)
}
const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[
"reset",
"set_token_stream",
"token_stream",
"token_stream_mut",
"token_store",
"parse_tree_storage",
"clear_dfa",
"add_error_listener",
"remove_error_listeners",
"node",
"into_token_stream",
"into_token_store",
"into_parsed_file",
];
fn parser_public_rule_method_names(rule_names: &[String]) -> Vec<String> {
let mut used = GENERATED_PARSER_RESERVED_RULE_METHODS
.iter()
.map(|name| (*name).to_owned())
.collect::<BTreeSet<_>>();
rule_names
.iter()
.map(|rule| {
let base = rust_function_name(rule);
let name = unique_rule_method_name(&base, &used);
used.insert(name.clone());
name
})
.collect()
}
fn unique_rule_method_name(base: &str, used: &BTreeSet<String>) -> String {
if !used.contains(base) {
return base.to_owned();
}
let plain = base.strip_prefix("r#").unwrap_or(base);
let reserved_collision = GENERATED_PARSER_RESERVED_RULE_METHODS.contains(&base);
let stem = if reserved_collision {
format!("{plain}_rule")
} else {
plain.to_owned()
};
let (mut candidate, mut suffix) = if reserved_collision {
(stem.clone(), 2)
} else {
(format!("{stem}_2"), 3)
};
while used.contains(&candidate) {
candidate = format!("{stem}_{suffix}");
suffix += 1;
}
candidate
}
/// Everything embedded mode contributes to the rendered parser module.
#[derive(Debug, Default)]
struct EmbeddedParserData {
/// ATN action state -> translated inline Rust statements.
inline_actions: BTreeMap<usize, String>,
/// (rule, pred) -> (translated expr, optional `<fail=...>` message).
predicates: BTreeMap<(usize, usize), (String, Option<String>)>,
/// rule -> translated `@init` statements (run at rule entry).
init_entry: BTreeMap<usize, String>,
/// rule -> translated `@after` statements (run before `finish_rule`).
after: BTreeMap<usize, String>,
/// rule -> whether the rule declares args/returns/locals.
rule_has_attrs: Vec<bool>,
/// Rule-transition source state -> translated caller arg expression.
call_args: BTreeMap<usize, String>,
/// rule -> escaped name of its first declared arg, if any.
rule_arg0: Vec<Option<String>>,
/// Rendered `__RuleAttrsN` struct definitions.
attrs_structs: String,
/// Member fields lowered onto the parser struct.
struct_fields: String,
field_inits: String,
/// `@members` fn items + generated facades for the parser impl block.
impl_items: String,
/// `@members` structs/impls and generated support types.
module_items: String,
/// Decisions the ANTLR tool would compile to `adaptivePredict` calls
/// (non-LL(1) per [`tool_decision_analysis`]); the generated code must
/// route these through the simulator on every visit.
adaptive_decisions: BTreeSet<usize>,
/// Tool LOOK(1) dispatch intervals for LL(1)-disjoint decisions.
ll1_decision_arms: BTreeMap<usize, Vec<Vec<(i32, i32)>>>,
}
/// Raw grammar-local booleans whose actions and predicates are portable
/// without executing target-language code.
#[derive(Debug, Default)]
struct PortableLocalData {
/// Rule index -> generated local variable declarations.
declarations: Vec<Vec<String>>,
/// ATN action source state -> generated assignment.
inline_actions: BTreeMap<usize, String>,
/// Parser predicate coordinate -> generated boolean expression.
predicates: BTreeMap<(usize, usize), (String, Option<String>)>,
/// Rules whose local state has no equivalent interpreted representation.
required_generated_rules: BTreeSet<usize>,
}
impl PortableLocalData {
fn has_semantics(&self) -> bool {
!self.required_generated_rules.is_empty()
}
fn step_render(&self) -> Option<PortableLocalStepRender<'_>> {
self.has_semantics().then_some(PortableLocalStepRender {
declarations: &self.declarations,
inline_actions: &self.inline_actions,
predicates: &self.predicates,
required_generated_rules: &self.required_generated_rules,
})
}
}
fn portable_local_name(name: &str) -> String {
format!("__antlr_local_{name}")
}
fn parse_portable_bool_assignment(body: &str) -> Option<(&str, bool)> {
let body = body.trim();
let body = body.strip_suffix(';').unwrap_or(body);
let (target, value) = body.split_once('=')?;
let target = target.trim().strip_prefix('$')?;
let value = match value.trim() {
"true" => true,
"false" => false,
_ => return None,
};
(!target.is_empty()
&& target
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()))
.then_some((target, value))
}
fn parse_portable_bool_predicate(body: &str) -> Option<(&str, bool)> {
let body = body.trim();
let (body, negated) = body
.strip_prefix('!')
.map_or((body, false), |body| (body.trim(), true));
let name = body.strip_prefix('$')?.trim();
(!name.is_empty()
&& name
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()))
.then_some((name, negated))
}
fn build_portable_local_data(
data: &InterpData,
source: &str,
patterns: &SemPatternFile,
) -> io::Result<PortableLocalData> {
let model = embedded::parse_embedded_rules_model(source, &data.rule_names);
let mut out = PortableLocalData {
declarations: vec![Vec::new(); data.rule_names.len()],
..PortableLocalData::default()
};
let mut local_names = Vec::with_capacity(data.rule_names.len());
for (rule_index, rule) in model.rules.iter().enumerate() {
let mut names = BTreeMap::new();
for attr in rule.attrs.iter().filter(|attr| attr.ty == "bool") {
let initial = if rule.local_names.contains(&attr.name) {
"false"
} else if rule.arg_names.first() == Some(&attr.name) {
"__precedence != 0"
} else {
continue;
};
let local = portable_local_name(&attr.name);
out.declarations[rule_index].push(format!("let mut {local} = {initial};"));
names.insert(attr.name.clone(), local);
}
local_names.push(names);
}
let action_slots = parser_action_source_block_slots(source, &data.rule_names);
for (state, slot) in
assign_states_to_parser_action_slots_with_model(data, &model, action_slots)?
{
let Some((name, value)) = parse_portable_bool_assignment(&slot.body) else {
continue;
};
let Some(local) = local_names
.get(slot.rule_index)
.and_then(|names| names.get(name))
else {
continue;
};
if patterns
.coordinate_disposition(
SemanticsKind::ParserAction,
data.rule_names.get(slot.rule_index).map(String::as_str),
None,
Some(state),
)
.is_some()
{
continue;
}
out.inline_actions
.insert(state, format!("{local} = {value};"));
out.required_generated_rules.insert(slot.rule_index);
}
let coordinates = parser_predicate_transitions(data)?;
let mut predicate_index = 0;
let mut offset = 0;
while let Some(block) = next_predicate_action_block(source, offset) {
offset = block.after_brace;
if !predicate_block_included(source, block.open_brace, &data.rule_names) {
continue;
}
let Some(&(rule_index, pred_index)) = coordinates.get(predicate_index) else {
break;
};
predicate_index += 1;
let Some((name, negated)) = parse_portable_bool_predicate(block.body) else {
continue;
};
let Some(local) = local_names
.get(rule_index)
.and_then(|names| names.get(name))
else {
continue;
};
if patterns
.coordinate_disposition(
SemanticsKind::ParserPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
None,
)
.is_some()
{
continue;
}
let expression = if negated {
format!("!{local}")
} else {
local.clone()
};
out.predicates.insert(
(rule_index, pred_index),
(
expression,
predicate_fail_message(source, block.after_brace),
),
);
out.required_generated_rules.insert(rule_index);
}
Ok(out)
}
/// LOOK(1) of one decision alternative, plus whether the walk crossed a
/// predicate — Java nulls such alternatives, forcing the decision adaptive.
#[derive(Debug, Default)]
struct DecisionAltLook {
symbols: BTreeSet<i32>,
hit_pred: bool,
}
/// Ports the ANTLR tool's `AnalysisPipeline` classification: the decisions
/// whose alternatives' LOOK(1) sets are not pairwise disjoint (or hit a
/// predicate, or come up empty). Java compiles LL(1)-disjoint decisions to
/// plain token switches — no simulator, no learned DFA — and routes every
/// other decision through `adaptivePredict`, the only place DFA states are
/// learned and full-context diagnostics fire. `dumpDFA` and diagnostic
/// output therefore only match Java when the generated routing agrees.
fn tool_decision_analysis(data: &InterpData) -> io::Result<ToolDecisionAnalysis> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut analysis = ToolDecisionAnalysis::default();
for (decision, state_number) in atn.decision_to_state().iter().enumerate() {
let Some(state) = atn.state(state_number) else {
continue;
};
// The tool never LL(1)-compiles non-greedy or left-recursion
// precedence decisions, disjoint LOOK or not — Java always emits
// `adaptivePredict` for them (a token switch would make a
// non-greedy loop greedy).
if state.non_greedy() || state.precedence_rule_decision() {
analysis.adaptive_decisions.insert(decision);
continue;
}
let looks: Vec<DecisionAltLook> = state
.transitions()
.iter()
.map(|transition| {
let mut look = DecisionAltLook::default();
let mut walk = DecisionLookWalk {
atn: &atn,
busy: BTreeSet::new(),
called_rules: vec![false; atn.rule_to_start_state().len()],
};
walk.walk(transition.target(), &mut Vec::new(), &mut look);
look
})
.collect();
if decision_alt_looks_disjoint(&looks) {
analysis.ll1_decision_arms.insert(
decision,
looks
.iter()
.map(|look| symbol_intervals(&look.symbols))
.collect(),
);
} else {
analysis.adaptive_decisions.insert(decision);
}
}
Ok(analysis)
}
/// Tool-side decision classification (see [`tool_decision_analysis`]).
#[derive(Debug, Default)]
struct ToolDecisionAnalysis {
/// Decisions Java compiles to `adaptivePredict` calls.
adaptive_decisions: BTreeSet<usize>,
/// Complete LOOK(1) dispatch intervals per alt for LL(1)-disjoint
/// decisions — exit alternatives included, unlike the within-rule
/// fast-path/LL(1) analyses, so legit input never falls through to the
/// simulator (Java's switch never does).
ll1_decision_arms: BTreeMap<usize, Vec<Vec<(i32, i32)>>>,
}
/// Collapses a sorted symbol set into inclusive intervals.
fn symbol_intervals(symbols: &BTreeSet<i32>) -> Vec<(i32, i32)> {
let mut intervals: Vec<(i32, i32)> = Vec::new();
for &symbol in symbols {
match intervals.last_mut() {
Some((_, stop)) if *stop + 1 == symbol => *stop = symbol,
_ => intervals.push((symbol, symbol)),
}
}
intervals
}
/// `AnalysisPipeline.disjoint`: pairwise-disjoint alt LOOK sets, with
/// Java's `getDecisionLookahead` nulling (empty or predicate-hitting sets)
/// folded in as an immediate non-LL(1) verdict.
fn decision_alt_looks_disjoint(looks: &[DecisionAltLook]) -> bool {
let mut combined = BTreeSet::new();
for look in looks {
if look.hit_pred || look.symbols.is_empty() {
return false;
}
if look.symbols.intersection(&combined).next().is_some() {
return false;
}
combined.extend(look.symbols.iter().copied());
}
true
}
struct DecisionLookWalk<'a> {
atn: &'a ParserAtn,
/// Java's `lookBusy`: (state, calling context) pairs already expanded.
busy: BTreeSet<(usize, Vec<usize>)>,
/// Java's `calledRuleStack`: rules on the walk's invocation path, the
/// recursion guard that bounds the context stack.
called_rules: Vec<bool>,
}
impl DecisionLookWalk<'_> {
/// `LL1Analyzer._LOOK` with an initially empty context: nested rule
/// invocations return precisely through `ctx`; the decision's own rule
/// boundary falls through the rule-stop state's return edges — the
/// deserializer materializes one per call site, which is exactly the
/// context-free FOLLOW Java walks there.
fn walk(&mut self, state_number: usize, ctx: &mut Vec<usize>, look: &mut DecisionAltLook) {
if !self.busy.insert((state_number, ctx.clone())) {
return;
}
let Some(state) = self.atn.state(state_number) else {
return;
};
if state.kind() == AtnStateKind::RuleStop {
if let Some(return_state) = ctx.pop() {
let cleared = state
.rule_index()
.map(|rule| std::mem::replace(&mut self.called_rules[rule], false));
self.walk(return_state, ctx, look);
if let (Some(rule), Some(flag)) = (state.rule_index(), cleared) {
self.called_rules[rule] = flag;
}
ctx.push(return_state);
return;
}
if state.transitions().is_empty() {
// The walk escaped through a stop state with no call sites —
// the start rule's end, where the only lookahead is EOF.
look.symbols.insert(TOKEN_EOF);
return;
}
}
for transition in state.transitions() {
match transition.data() {
ParserTransitionData::Rule {
target,
rule_index,
follow_state,
..
} => {
if self.called_rules.get(rule_index).copied().unwrap_or(true) {
continue;
}
self.called_rules[rule_index] = true;
ctx.push(follow_state);
self.walk(target, ctx, look);
ctx.pop();
self.called_rules[rule_index] = false;
}
ParserTransitionData::Predicate { .. }
| ParserTransitionData::Precedence { .. } => {
look.hit_pred = true;
}
ParserTransitionData::Epsilon { target }
| ParserTransitionData::Action { target, .. } => {
self.walk(target, ctx, look);
}
_ => {
// Terminal edge: enumerate the vocabulary through the
// shared `matches` so atoms, ranges, sets, negations and
// wildcards agree with the simulator exactly.
for symbol in TOKEN_EOF..=self.atn.max_token_type() {
if transition.matches(symbol, 1, self.atn.max_token_type()) {
look.symbols.insert(symbol);
}
}
}
}
}
}
}
/// Builds the embedded translation of every action, predicate, `@init` and
/// `@after` body in the rendered grammar, plus the members model.
fn build_embedded_parser_data(
data: &InterpData,
source: &str,
type_name: &str,
grammar_name: &str,
) -> io::Result<EmbeddedParserData> {
let model = embedded::parse_embedded_model(source, &data.rule_names)?;
let token_types: BTreeMap<String, i32> = data
.symbolic_names
.iter()
.enumerate()
.filter_map(|(token_type, name)| {
let name = name.as_ref()?;
i32::try_from(token_type)
.ok()
.map(|token_type| (name.clone(), token_type))
})
.collect();
let finish_body = |body: &str, translated: &str| -> String {
post_process_embedded(body, translated, type_name)
};
let decision_analysis = tool_decision_analysis(data)?;
let mut out = EmbeddedParserData {
rule_has_attrs: model
.rules
.iter()
.map(embedded::RuleModel::has_attrs)
.collect(),
adaptive_decisions: decision_analysis.adaptive_decisions,
ll1_decision_arms: decision_analysis.ll1_decision_arms,
..EmbeddedParserData::default()
};
// Mid-rule embedded actions: pair every action block with its ATN action
// state using the same per-rule authored-block slot walk as the manifest.
let slots = parser_action_source_block_slots(source, &data.rule_names);
for (state, slot) in assign_states_to_parser_action_slots_with_model(data, &model, slots)? {
if slot.body.trim().is_empty() {
out.inline_actions.insert(state, String::new());
continue;
}
let ctx = embedded::TranslationCtx {
model: &model,
rule_index: slot.rule_index,
body_offset: Some(slot.source_offset),
site: embedded::ActionSite::Body,
token_types: &token_types,
};
let translated = embedded::translate_body(&slot.body, &ctx)?;
out.inline_actions
.insert(state, finish_body(&slot.body, &translated));
}
// Predicates: same source walk/coordinate pairing as non-embedded mode.
let coordinates = parser_predicate_transitions(data)?;
let mut predicate_index = 0;
let mut pred_offset = 0;
while let Some(block) = next_predicate_action_block(source, pred_offset) {
pred_offset = block.after_brace;
if !predicate_block_included(source, block.open_brace, &data.rule_names) {
continue;
}
let Some(&(rule_index, pred_index)) = coordinates.get(predicate_index) else {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"embedded predicate {{{}}}? has no parser ATN predicate transition",
block.body
),
));
};
predicate_index += 1;
let ctx = embedded::TranslationCtx {
model: &model,
rule_index,
body_offset: Some(block.open_brace),
site: embedded::ActionSite::Body,
token_types: &token_types,
};
let translated = embedded::translate_body(block.body.trim(), &ctx)?;
let message = predicate_fail_message(source, block.after_brace);
out.predicates.insert(
(rule_index, pred_index),
(finish_body(block.body, &translated), message),
);
}
// @init / @after bodies from the rule headers.
for (rule_index, rule) in model.rules.iter().enumerate() {
if let Some(body) = &rule.init_body {
let ctx = embedded::TranslationCtx {
model: &model,
rule_index,
body_offset: None,
site: embedded::ActionSite::Init,
token_types: &token_types,
};
let translated = embedded::translate_body(body, &ctx)?;
out.init_entry
.insert(rule_index, finish_body(body, &translated));
}
if let Some(body) = &rule.after_body {
let ctx = embedded::TranslationCtx {
model: &model,
rule_index,
body_offset: None,
site: embedded::ActionSite::After,
token_types: &token_types,
};
let translated = embedded::translate_body(body, &ctx)?;
out.after.insert(rule_index, finish_body(body, &translated));
}
}
// Per-rule attrs structs — emitted for every rule (context views
// reference them uniformly; attr-less rules get an empty struct).
for (rule_index, rule) in model.rules.iter().enumerate() {
let struct_name = embedded::attrs_struct_name(rule_index);
let mut fields = String::new();
for attr in &rule.attrs {
let _ = writeln!(
fields,
" pub {}: {},",
embedded::escape_keyword(&attr.name),
attr.ty
);
}
let _ = writeln!(
out.attrs_structs,
"#[derive(Clone, Debug, Default)]\n#[allow(non_snake_case, dead_code)]\npub struct {struct_name} {{\n{fields}}}\n"
);
}
// Members: fields, impl items, module items.
out.struct_fields
.push_str(" __embedded_pending_arg: Option<i64>,\n");
out.field_inits
.push_str(" __embedded_pending_arg: None,\n");
for field in &model.parser_members.fields {
let _ = writeln!(out.struct_fields, " {}: {},", field.name, field.ty);
let _ = writeln!(
out.field_inits,
" {}: {},",
field.name, field.init
);
}
for item in &model.parser_members.impl_items {
let item = post_process_embedded(item, item, type_name);
let mut indented = String::with_capacity(item.len());
for (line_index, line) in item.lines().enumerate() {
if line_index > 0 {
indented.push_str("\n ");
}
indented.push_str(line);
}
let _ = writeln!(out.impl_items, " {indented}\n");
}
for item in &model.parser_members.module_items {
let item = post_process_embedded(item, item, type_name);
let _ = writeln!(out.module_items, "{item}\n");
}
// Rule-call argument expressions, correlated to rule transitions the
// same way parser_rule_args does (source order per callee rule).
out.call_args = embedded_rule_call_args(data, source, &model)?;
out.rule_arg0 = model
.rules
.iter()
.map(|rule| {
rule.arg_names
.first()
.map(|name| embedded::escape_keyword(name))
})
.collect();
// Associated token constants: rendered bodies reference tokens as
// `TParser::NL` (post-processed to `Self::NL`).
for (name, token_type) in &token_types {
let _ = writeln!(
out.impl_items,
" #[allow(dead_code)]\n pub const {name}: i32 = {token_type};"
);
}
out.impl_items.push('\n');
// Recognizer-surface facades the rendered bodies call.
out.impl_items.push_str(&embedded_parser_facades());
out.module_items.push_str(EMBEDDED_INPUT_FACADE);
out.module_items
.push_str(&render_embedded_context_types(grammar_name, data, &model));
Ok(out)
}
/// Scans `rule[expr]` call sites in the rendered grammar and pairs each with
/// its ATN rule-transition source state (source order per callee rule, the
/// same correlation as `parser_rule_args`). Supports integer literals and
/// single identifiers (translated to the caller's `__attrs` field).
fn embedded_rule_call_args(
data: &InterpData,
source: &str,
model: &embedded::EmbeddedModel,
) -> io::Result<BTreeMap<usize, String>> {
let mut calls: Vec<(usize, usize, String)> = Vec::new();
for (rule_index, rule_name) in data.rule_names.iter().enumerate() {
let pattern = format!("{rule_name}[");
let mut offset = 0;
while let Some(start) = source[offset..].find(&pattern).map(|index| offset + index) {
let value_start = start + pattern.len();
let Some(value_stop) = source[value_start..]
.find(']')
.map(|index| value_start + index)
else {
break;
};
offset = value_stop + 1;
if start != 0
&& source[..start]
.chars()
.next_back()
.is_some_and(|ch| ch == '_' || ch.is_ascii_alphanumeric())
{
continue;
}
let value = source[value_start..value_stop].trim();
let expression = if value.parse::<i64>().is_ok() {
Some(value.to_owned())
} else if value
.chars()
.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
&& value
.chars()
.next()
.is_some_and(|ch| ch == '_' || ch.is_ascii_alphabetic())
{
// A bare identifier is a caller attribute (arg/local/return).
Some(format!("__attrs.{}", embedded::escape_keyword(value)))
} else {
None
};
if let Some(expression) = expression {
calls.push((start, rule_index, expression));
}
}
}
let _ = model;
calls.sort_by_key(|(start, _, _)| *start);
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut rule_transitions = Vec::new();
for state in atn.states() {
for transition in state.transitions() {
if let ParserTransitionData::Rule { rule_index, .. } = transition.data() {
rule_transitions.push((state.state_number(), rule_index));
}
}
}
let mut used = vec![false; rule_transitions.len()];
let mut args = BTreeMap::new();
for (_, rule_index, expression) in calls {
if let Some((index, (source_state, _))) = rule_transitions
.iter()
.enumerate()
.find(|(index, (_, transition_rule))| !used[*index] && *transition_rule == rule_index)
{
used[index] = true;
args.insert(*source_state, expression);
}
}
Ok(args)
}
/// Generates the typed context views, listener trait, and walker for the
/// embedded `.test.stg` surface:
///
/// * one `<Rule>Context` view per parser rule (plus one per labeled
/// alternative) with positional child accessors (`ctx.e(0)` /
/// `ctx.e_all()`, `ctx.INT(0)` / `ctx.INT_all()`), `child_count()`,
/// `start()`, public attribute fields, and a `FromRuleNode` impl backing
/// `ctx.downcast_ref::<XContext>()`;
/// * the `<Grammar>Listener` trait with defaulted `enter_/exit_<rule>` (and
/// per-labeled-alternative) callbacks plus terminal/error-node visitors;
/// * a module-local `ParseTreeWalker` whose bridge dispatches the runtime
/// walker onto the typed listener callbacks, threading the invoking-state
/// chain Java's `RuleContext.toString` renders (`[13 6]`).
fn render_embedded_context_types(
grammar_name: &str,
data: &InterpData,
model: &embedded::EmbeddedModel,
) -> String {
let mut out = String::new();
let listener_trait = format!(
"{}Listener",
grammar_name.strip_suffix("Parser").unwrap_or(grammar_name)
);
let token_accessors: Vec<(String, i32)> = data
.symbolic_names
.iter()
.enumerate()
.filter_map(|(token_type, name)| {
let name = name.as_ref()?;
i32::try_from(token_type).ok().map(|ty| (name.clone(), ty))
})
.collect();
out.push_str(
r#"#[allow(dead_code)]
#[derive(Clone)]
pub struct TerminalNode<'a> {
__node: RuntimeTerminalNode<'a>,
}
#[allow(dead_code)]
impl<'a> TerminalNode<'a> {
fn new(node: RuntimeTerminalNode<'a>) -> Self {
Self { __node: node }
}
pub fn symbol(&self) -> antlr4_runtime::TokenView<'a> {
self.__node.symbol()
}
}
impl std::fmt::Display for TerminalNode<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.__node.text())
}
}
#[allow(dead_code)]
#[derive(Clone)]
pub struct ErrorNode<'a> {
__node: RuntimeErrorNode<'a>,
}
#[allow(dead_code)]
impl<'a> ErrorNode<'a> {
fn new(node: RuntimeErrorNode<'a>) -> Self {
Self { __node: node }
}
pub fn symbol(&self) -> antlr4_runtime::TokenView<'a> {
self.__node.symbol()
}
}
impl std::fmt::Display for ErrorNode<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.__node.text())
}
}
#[allow(dead_code)]
#[derive(Clone)]
enum __GeneratedRuleContext<'a> {
Stored(RuleNodeView<'a>),
Active {
context: &'a antlr4_runtime::ParserRuleContext,
storage: &'a antlr4_runtime::ParseTreeStorage,
tokens: &'a antlr4_runtime::TokenStore,
},
}
#[allow(dead_code)]
trait __FromActiveRuleContext<'a>: Sized {
fn __from_active(
context: &'a antlr4_runtime::ParserRuleContext,
invocation_states: Vec<isize>,
storage: &'a antlr4_runtime::ParseTreeStorage,
tokens: &'a antlr4_runtime::TokenStore,
) -> Option<Self>;
}
#[allow(dead_code)]
fn __active_context_view<'a, T: __FromActiveRuleContext<'a>>(
context: &'a antlr4_runtime::ParserRuleContext,
invocation_states: Vec<isize>,
storage: &'a antlr4_runtime::ParseTreeStorage,
tokens: &'a antlr4_runtime::TokenStore,
) -> Option<T> {
T::__from_active(context, invocation_states, storage, tokens)
}
"#,
);
// (struct name, rule index, alt label) — one per rule plus one per
// labeled alternative.
let mut views: Vec<(String, usize)> = Vec::new();
for (rule_index, rule) in model.rules.iter().enumerate() {
views.push((format!("{}Context", rust_type_name(&rule.name)), rule_index));
for alt in &rule.alts {
if let Some(label) = &alt.label {
let view = format!("{}Context", rust_type_name(label));
if !views.iter().any(|(existing, _)| *existing == view) {
views.push((view, rule_index));
}
}
}
}
for (view_name, rule_index) in &views {
let rule = &model.rules[*rule_index];
let attrs_struct = embedded::attrs_struct_name(*rule_index);
let mut fields = String::new();
let mut field_inits = String::new();
for attr in &rule.attrs {
let name = embedded::escape_keyword(&attr.name);
let _ = writeln!(fields, " pub {name}: {ty},", ty = attr.ty);
let _ = writeln!(field_inits, " {name}: __attrs.{name}.clone(),");
}
let _ = writeln!(
out,
"#[allow(non_camel_case_types, dead_code)]\n#[derive(Clone)]\npub struct {view_name}<'a> {{\n __node: __GeneratedRuleContext<'a>,\n __invocation_states: Vec<isize>,\n{fields}}}\n"
);
let _ = writeln!(
out,
"impl<'a> FromRuleNode<'a> for {view_name}<'a> {{\n fn from_rule_node(node: RuleNodeView<'a>) -> Option<Self> {{\n if node.rule_index() != {rule_index} {{ return None; }}\n Some(Self::__from_node(node))\n }}\n}}\n\nimpl<'a> __FromActiveRuleContext<'a> for {view_name}<'a> {{\n fn __from_active(\n context: &'a antlr4_runtime::ParserRuleContext,\n invocation_states: Vec<isize>,\n storage: &'a antlr4_runtime::ParseTreeStorage,\n tokens: &'a antlr4_runtime::TokenStore,\n ) -> Option<Self> {{\n if context.rule_index() != {rule_index} {{ return None; }}\n let __default = {attrs_struct}::default();\n let __attrs = context.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Some(Self {{\n __node: __GeneratedRuleContext::Active {{ context, storage, tokens }},\n __invocation_states: invocation_states,\n{field_inits} }})\n }}\n}}\n"
);
let mut accessors = String::new();
let _ = writeln!(
accessors,
" fn __from_node(node: RuleNodeView<'a>) -> Self {{\n let invocation_states = node.invocation_states().collect();\n Self::__from_node_with_invocation_states(node, invocation_states)\n }}\n\n fn __from_child_node(node: RuleNodeView<'a>, parent_invocation_states: &[isize]) -> Self {{\n let mut invocation_states = Vec::with_capacity(parent_invocation_states.len() + 1);\n invocation_states.push(node.invoking_state());\n invocation_states.extend_from_slice(parent_invocation_states);\n Self::__from_node_with_invocation_states(node, invocation_states)\n }}\n\n fn __from_listener_node(node: RuleNodeView<'a>, invocation_states: Option<&[isize]>) -> Self {{\n invocation_states.map_or_else(\n || Self::__from_node(node),\n |states| Self::__from_node_with_invocation_states(node, states.to_vec()),\n )\n }}\n\n fn __from_node_with_invocation_states(node: RuleNodeView<'a>, invocation_states: Vec<isize>) -> Self {{\n let __default = {attrs_struct}::default();\n let __attrs = node.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Self {{\n __node: __GeneratedRuleContext::Stored(node),\n __invocation_states: invocation_states,\n{field_inits} }}\n }}\n"
);
// A grammar rule claiming a built-in helper's name (`start`,
// `child_count`) takes the accessor slot; the built-in yields.
let rule_claims = |name: &str| {
model
.rules
.iter()
.any(|rule| rust_function_name(&rule.name) == name)
};
if !rule_claims("child_count") {
let _ = writeln!(
accessors,
" pub fn child_count(&self) -> usize {{\n match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_count(),\n __GeneratedRuleContext::Active {{ context, .. }} => context.child_count(),\n }}\n }}"
);
}
if !rule_claims("start") {
let _ = writeln!(
accessors,
" pub fn start(&self) -> __GeneratedTokenView {{\n let token = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.start(),\n __GeneratedRuleContext::Active {{ context, tokens, .. }} => context.start(tokens),\n }};\n __GeneratedTokenView {{ text: token.map(|token| token.text().to_owned()).unwrap_or_default() }}\n }}"
);
}
for (child_index, child) in model.rules.iter().enumerate() {
let method = rust_function_name(&child.name);
let child_view = format!("{}Context", rust_type_name(&child.name));
let _ = writeln!(
accessors,
" pub fn {method}(&self, index: usize) -> {child_view}<'a> {{\n let node = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_rules({child_index}).nth(index),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_rules(storage, tokens, {child_index}).nth(index),\n }}.expect(\"missing rule child\");\n {child_view}::__from_child_node(node, &self.__invocation_states)\n }}\n pub fn {method}_all(&self) -> Vec<{child_view}<'a>> {{\n let nodes: Vec<_> = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_rules({child_index}).collect(),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_rules(storage, tokens, {child_index}).collect(),\n }};\n nodes.into_iter().map(|node| {child_view}::__from_child_node(node, &self.__invocation_states)).collect()\n }}"
);
}
for (token_name, token_type) in &token_accessors {
let _ = writeln!(
accessors,
" #[allow(non_snake_case)]\n pub fn {token_name}(&self, index: usize) -> TerminalNode<'a> {{\n let node = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_tokens({token_type}).nth(index),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_tokens(storage, tokens, {token_type}).nth(index),\n }}.expect(\"missing token child\");\n TerminalNode::new(node)\n }}\n #[allow(non_snake_case)]\n pub fn {token_name}_all(&self) -> Vec<TerminalNode<'a>> {{\n let nodes: Vec<_> = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_tokens({token_type}).collect(),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_tokens(storage, tokens, {token_type}).collect(),\n }};\n nodes.into_iter().map(TerminalNode::new).collect()\n }}"
);
}
let _ = writeln!(
out,
"#[allow(dead_code, clippy::all)]\nimpl<'a> {view_name}<'a> {{\n{accessors}}}\n"
);
// Java's RuleContext.toString(): bracketed invoking-state chain from
// this context to the root, the root's sentinel excluded.
let _ = writeln!(
out,
"impl std::fmt::Display for {view_name}<'_> {{\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{\n let chain: Vec<String> = self.__invocation_states.iter().map(|state| state.to_string()).collect();\n write!(f, \"[{{}}]\", chain.join(\" \"))\n }}\n}}\n"
);
}
// Listener trait with defaulted callbacks. Method names are snake_case
// per the `.test.stg` convention (`exit_call` for `# Call`); the `r#`
// keyword escape is dropped because the composed `enter_x`/`exit_x`
// identifier is never a keyword.
let listener_method =
|name: &str| -> String { rust_function_name(name).trim_start_matches("r#").to_owned() };
let mut trait_methods = String::new();
let mut enter_arms = String::new();
let mut exit_arms = String::new();
for (rule_index, rule) in model.rules.iter().enumerate() {
let mut names: Vec<(String, String)> = vec![(
listener_method(&rule.name),
format!("{}Context", rust_type_name(&rule.name)),
)];
for alt in &rule.alts {
if let Some(label) = &alt.label {
let pair = (
listener_method(label),
format!("{}Context", rust_type_name(label)),
);
if !names.contains(&pair) {
names.push(pair);
}
}
}
for (method, view) in &names {
let _ = writeln!(
trait_methods,
" fn enter_{method}(&mut self, _ctx: &{view}) {{}}\n fn exit_{method}(&mut self, _ctx: &{view}) {{}}"
);
}
// Dispatch: an unlabeled rule fires its plain callbacks; a rule with
// labeled alternatives fires the callback of the alternative that
// matched. Left-recursive operator alternatives are identified
// structurally (their first child is the rule itself), matching
// ANTLR's per-alternative context classes.
let op_labels: Vec<String> = rule
.alts
.iter()
.filter(|alt| alt.is_lr_operator(&rule.name))
.filter_map(|alt| alt.label.clone())
.collect();
let primary_labels: Vec<String> = rule
.alts
.iter()
.filter(|alt| !alt.is_lr_operator(&rule.name))
.filter_map(|alt| alt.label.clone())
.collect();
let has_labels = names.len() > 1;
let dispatch = |phase: &str| -> String {
if !has_labels {
let (method, view) = &names[0];
return format!(
" self.0.{phase}_{method}(&{view}::__from_listener_node(context, self.1.as_deref()));\n"
);
}
let mut out = String::new();
let op = op_labels
.first()
.filter(|_| op_labels.iter().collect::<BTreeSet<_>>().len() == 1);
let primary = primary_labels
.first()
.filter(|_| primary_labels.iter().collect::<BTreeSet<_>>().len() == 1);
match (op, primary) {
(Some(op), Some(primary)) => {
let op_method = listener_method(op);
let op_view = format!("{}Context", rust_type_name(op));
let primary_method = listener_method(primary);
let primary_view = format!("{}Context", rust_type_name(primary));
let _ = writeln!(
out,
" if context.child_rule_trees({rule_index}).next().is_some() {{ self.0.{phase}_{op_method}(&{op_view}::__from_listener_node(context, self.1.as_deref())); }} else {{ self.0.{phase}_{primary_method}(&{primary_view}::__from_listener_node(context, self.1.as_deref())); }}"
);
}
_ => {
// No unambiguous per-alternative identity available; fire
// the rule-level callback only.
let (method, view) = &names[0];
let _ = writeln!(
out,
" self.0.{phase}_{method}(&{view}::__from_listener_node(context, self.1.as_deref()));"
);
}
}
out
};
let enter_calls = dispatch("enter");
let exit_calls = dispatch("exit");
let _ = writeln!(
enter_arms,
" {rule_index} => {{\n{enter_calls} }}"
);
let _ = writeln!(
exit_arms,
" {rule_index} => {{\n{exit_calls} }}"
);
}
let _ = writeln!(
out,
"#[allow(dead_code, unused_variables)]\npub trait {listener_trait} {{\n{trait_methods} fn visit_terminal(&mut self, _node: &TerminalNode) {{}}\n fn visit_error_node(&mut self, _node: &ErrorNode) {{}}\n fn output(&mut self) -> std::io::Stdout {{ std::io::stdout() }}\n}}\n"
);
// Bridge + module-local walker (shadows the runtime walker so rendered
// `ParseTreeWalker::walk(&mut listener, tree)` dispatches typed
// callbacks.
let _ = writeln!(
out,
r#"#[allow(dead_code)]
struct __ListenerBridge<'a, T: {listener_trait}>(&'a mut T, Option<Vec<isize>>);
impl<T: {listener_trait}> antlr4_runtime::ParseTreeListener for __ListenerBridge<'_, T> {{
fn enter_every_rule(&mut self, context: RuleNodeView<'_>) -> Result<(), antlr4_runtime::AntlrError> {{
if let Some(invocation_states) = &mut self.1 {{
invocation_states.insert(0, context.invoking_state());
}}
match context.rule_index() {{
{enter_arms} _ => {{}}
}}
Ok(())
}}
fn exit_every_rule(&mut self, context: RuleNodeView<'_>) -> Result<(), antlr4_runtime::AntlrError> {{
match context.rule_index() {{
{exit_arms} _ => {{}}
}}
if let Some(invocation_states) = &mut self.1 {{
invocation_states.remove(0);
}}
Ok(())
}}
fn visit_terminal(&mut self, node: RuntimeTerminalNode<'_>) -> Result<(), antlr4_runtime::AntlrError> {{
self.0.visit_terminal(&TerminalNode::new(node));
Ok(())
}}
fn visit_error_node(&mut self, node: RuntimeErrorNode<'_>) -> Result<(), antlr4_runtime::AntlrError> {{
self.0.visit_error_node(&ErrorNode::new(node));
Ok(())
}}
}}
#[allow(dead_code)]
pub struct ParseTreeWalker;
#[allow(dead_code)]
impl ParseTreeWalker {{
pub fn walk<T: {listener_trait}>(listener: &mut T, tree: antlr4_runtime::Node<'_>) {{
let mut bridge = __ListenerBridge(listener, None);
let _ = antlr4_runtime::ParseTreeWalker::walk(&mut bridge, tree);
}}
pub fn walk_with_invocation_states<T: {listener_trait}>(
listener: &mut T,
tree: antlr4_runtime::Node<'_>,
parent_invocation_states: Vec<isize>,
) {{
let mut bridge = __ListenerBridge(listener, Some(parent_invocation_states));
let _ = antlr4_runtime::ParseTreeWalker::walk(&mut bridge, tree);
}}
}}
"#
);
out
}
/// Post-translation cleanups shared by all embedded bodies: `TParser::NL`
/// becomes `Self::NL` so token constants resolve inside the generic impl.
fn post_process_embedded(_original: &str, translated: &str, type_name: &str) -> String {
let translated = replace_all(translated, &format!("{type_name}::"), "Self::");
let translated = replace_all(
&translated,
"(&__ctx).to_string_tree(Some(self), self.base.token_store())",
"(&__ctx).to_string_tree(Some(self), self.base.parse_tree_storage(), self.base.token_store())",
);
replace_all(
&translated,
"(&__ctx).to_string_tree(Some(self))",
"(&__ctx).to_string_tree(Some(self), self.base.parse_tree_storage(), self.base.token_store())",
)
}
/// Plain substring replacement (`str::replace` is a disallowed method here).
fn replace_all(text: &str, needle: &str, replacement: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(index) = rest.find(needle) {
out.push_str(&rest[..index]);
out.push_str(replacement);
rest = &rest[index + needle.len()..];
}
out.push_str(rest);
out
}
/// Generated-recognizer surface used by rendered test actions.
fn embedded_parser_facades() -> String {
r#" #[allow(dead_code)]
fn output(&mut self) -> std::io::Stdout {
std::io::stdout()
}
#[allow(dead_code)]
fn input(&mut self) -> __GeneratedInput<'_, L> {
__GeneratedInput(self.base.input())
}
#[allow(dead_code)]
fn interpreter(&mut self) -> &mut Self {
self
}
#[allow(dead_code)]
fn set_error_handler(&mut self, _strategy: antlr4_runtime::BailErrorStrategy) {
self.base.set_bail_on_error(true);
}
#[allow(dead_code)]
fn expected_tokens(&mut self) -> antlr4_runtime::ExpectedTokenSet {
self.base.expected_tokens_current(atn())
}
#[allow(dead_code)]
fn rule_invocation_stack(&self) -> Vec<String> {
self.base.rule_invocation_stack()
}
#[allow(dead_code)]
fn dump_dfa(&mut self) {
if let Some(simulator) = self.simulator.as_ref() {
print!("{}", simulator.dump_dfa_java_style(self.base.vocabulary()));
}
}
"#
.to_owned()
}
/// Token-stream facade matching the `.test.stg` surface
/// (`self.input().text()` / `.la(i)` / `.lt(i).text()`).
const EMBEDDED_INPUT_FACADE: &str = r#"
#[allow(dead_code)]
pub struct __GeneratedInput<'a, L: TokenSource>(&'a mut CommonTokenStream<L>);
#[allow(dead_code)]
impl<L: TokenSource> __GeneratedInput<'_, L> {
pub fn text(&mut self) -> String {
self.0.text_all()
}
pub fn la(&mut self, offset: isize) -> i32 {
antlr4_runtime::IntStream::la(self.0, offset)
}
pub fn lt(&mut self, offset: isize) -> __GeneratedTokenView {
__GeneratedTokenView {
text: self
.0
.lt(offset)
.map(|token| token.text().to_owned())
.unwrap_or_default(),
}
}
}
#[allow(dead_code)]
pub struct __GeneratedTokenView {
text: String,
}
#[allow(dead_code)]
impl __GeneratedTokenView {
pub fn text(&self) -> &str {
&self.text
}
}
"#;
/// Collects `assume-*`-overridden action states that should become explicit
/// empty `run_action` arms instead of falling through to user hooks.
fn collect_noop_action_states(
data: &InterpData,
patterns: &SemPatternFile,
) -> io::Result<BTreeSet<usize>> {
let mut noop_action_states = BTreeSet::new();
let action_state_rules = parser_action_state_rules(data)?;
for state in action_state_rules.keys() {
if parser_action_assume_overridden(patterns, data, &action_state_rules, *state) {
noop_action_states.insert(*state);
}
}
Ok(noop_action_states)
}
/// Public per-rule entry methods (`pub fn s(&mut self) -> …`).
fn render_public_rule_methods(public_rule_method_names: &[String]) -> String {
let mut rule_methods = String::new();
for (index, rule_method_name) in public_rule_method_names.iter().enumerate() {
writeln!(
rule_methods,
" pub fn {rule_method_name}(&mut self) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{"
)
.expect("writing to a string cannot fail");
writeln!(rule_methods, " self.parse_rule({index})")
.expect("writing to a string cannot fail");
writeln!(rule_methods, " }}").expect("writing to a string cannot fail");
}
rule_methods
}
/// Step-render view over the embedded data.
///
/// `force_adaptive` stays off: per-decision routing follows the tool
/// classification in `adaptive_decisions` instead, matching which decisions
/// Java compiles to switches versus `adaptivePredict` calls.
fn embedded_step_render(embedded: &EmbeddedParserData) -> EmbeddedStepRender<'_> {
EmbeddedStepRender {
force_adaptive: false,
adaptive_decisions: &embedded.adaptive_decisions,
ll1_decision_arms: &embedded.ll1_decision_arms,
predicates: &embedded.predicates,
rule_has_attrs: &embedded.rule_has_attrs,
init_entry: &embedded.init_entry,
after: &embedded.after,
call_args: &embedded.call_args,
rule_arg0: &embedded.rule_arg0,
}
}
/// The module/struct/impl text an [`EmbeddedParserData`] contributes to the
/// rendered parser, empty in template mode.
fn embedded_render_slots(
embedded_data: Option<&EmbeddedParserData>,
) -> (String, String, String, String, String) {
embedded_data.map_or_else(
|| {
(
String::new(),
String::new(),
String::new(),
String::new(),
String::new(),
)
},
|embedded| {
(
embedded.attrs_structs.clone(),
embedded.module_items.clone(),
embedded.struct_fields.clone(),
embedded.field_inits.clone(),
embedded.impl_items.clone(),
)
},
)
}
fn render_parser_with_options(
grammar_name: &str,
data: &InterpData,
grammar_source: Option<&str>,
options: ParserRenderOptions<'_>,
) -> io::Result<String> {
let empty_patterns = SemPatternFile::default();
let patterns = options.patterns.unwrap_or(&empty_patterns);
let type_name = rust_type_name(grammar_name);
let metadata = render_parser_metadata(grammar_name, data);
let parser_atn_data = render_u32_slice(
AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?
.packed_words(),
);
let token_constants = render_token_constants(data);
let rule_constants = render_rule_constants(data);
// Embedded mode: the grammar carries real Rust action/predicate bodies
// (rendered through the target `.test.stg`); translate and splice them
// instead of recognizing template markup.
let embedded_data = if options.embedded {
let source = grammar_source.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"--actions embedded requires --grammar",
)
})?;
Some(build_embedded_parser_data(
data,
source,
&type_name,
grammar_name,
)?)
} else {
None
};
let embedded_step_render = embedded_data.as_ref().map(embedded_step_render);
let template_grammar_source = if options.embedded {
None
} else {
grammar_source
};
let mut portable_local_data = template_grammar_source
.map(|source| build_portable_local_data(data, source, patterns))
.transpose()?
.unwrap_or_default();
portable_local_data.required_generated_rules =
parser_rule_callers_reaching(data, &portable_local_data.required_generated_rules)?;
// A per-coordinate `assume-*` override is a documented no-op fallback; it
// must not fall through to `parser_action_hook`, which would fail loud under
// the Error policy or run a user side effect for a coordinate the manifest
// reports as ignored. A `hook`/`error`
// override falls through to the `parser_action_hook` catch-all, but an
// `assume-*` override gets an explicit empty arm.
let mut noop_action_states = collect_noop_action_states(data, patterns)?;
// ANTLR-synthesized action states (left-recursion elimination, etc.) are
// no-ops with no author intent. They must NOT reach `parser_action_hook`
// either, or they would fail loud at runtime under the Error policy — the
// same treatment `enforce_sem_unknown` gives them at codegen time. Give each
// an explicit empty `run_action` arm.
noop_action_states.extend(synthetic_parser_action_states(data, grammar_source)?);
// Authored empty action bodies are explicit no-ops too: they carry source
// provenance for the manifest but should not disable generated parsing or
// fall through to runtime hooks under strict semantic policies.
noop_action_states.extend(empty_parser_action_states(data, grammar_source)?);
let predicates = template_grammar_source.map_or_else(
// Without grammar source, per-coordinate `--sem-patterns` overrides still
// resolve by ATN coordinate, so honor them here too — otherwise a
// documented `.interp`-only override the manifest reports as active would
// have no runtime effect.
|| parser_predicate_templates_from_overrides(data, patterns),
|grammar| parser_predicate_templates(data, grammar, patterns),
)?;
let rule_args = template_grammar_source
.map_or_else(|| Ok(Vec::new()), |grammar| parser_rule_args(data, grammar))?;
let inline_action_statements = embedded_data.as_ref().map_or_else(
|| portable_local_data.inline_actions.clone(),
|embedded| embedded.inline_actions.clone(),
);
let inline_action_states = inline_action_statements
.keys()
.copied()
.collect::<BTreeSet<_>>();
let action_states = parser_action_states(data)?
.into_iter()
.collect::<BTreeSet<_>>();
let mut generated_action_states = if options.embedded {
action_states.clone()
} else {
// Synthetic actions and explicit assume-* overrides are no-op states.
// They should not disable generated parser rules just because they have
// an ATN action transition; real author actions still fall through to
// the interpreted path in non-embedded mode.
noop_action_states
.intersection(&action_states)
.copied()
.collect()
};
generated_action_states.extend(portable_local_data.inline_actions.keys().copied());
// Under a non-default unknown-coordinate policy every predicate transition
// must reach the interpreter (which applies the policy), so the coordinate
// inventory is read from the ATN even without grammar source.
let predicate_coordinates =
if grammar_source.is_some() || options.sem_unknown != SemUnknownPolicy::AssumeTrue {
parser_predicate_transitions(data)?
} else {
Vec::new()
}
.into_iter()
.collect::<BTreeSet<_>>();
let mut generated_predicate_coordinates = if options.embedded {
predicate_coordinates.clone()
} else {
predicates
.iter()
.filter_map(|(coordinate, predicate)| {
can_generate_parser_predicate(predicate).then_some(*coordinate)
})
.collect::<BTreeSet<_>>()
};
generated_predicate_coordinates.extend(portable_local_data.predicates.keys().copied());
let has_action_dispatch = !action_states.is_empty();
let has_predicate_dispatch = !predicates.is_empty();
let track_alt_numbers = grammar_source.is_some_and(uses_alt_number_contexts);
let generated_rule_enabled = vec![true; data.rule_names.len()];
let generated_rules = parser_generated_rules(
data,
&generated_rule_enabled,
&rule_args,
ActionStateSets {
all: &action_states,
generated: &generated_action_states,
inline: &inline_action_states,
},
PredicateCoordinateSets {
all: &predicate_coordinates,
generated: &generated_predicate_coordinates,
},
(has_action_dispatch || has_predicate_dispatch || portable_local_data.has_semantics())
&& !options.embedded,
)?;
require_portable_local_rules_generated(
&generated_rules,
&portable_local_data.required_generated_rules,
data,
)?;
if options.require_generated_parser || options.embedded {
// Embedded actions only run on the generated path, so every rule must
// compile; an interpreted fallback would silently skip them.
require_all_parser_rules_generated(&generated_rules, data)?;
}
let direct_generated_rule_calls = generated_rules
.iter()
.map(Option::is_some)
.collect::<Vec<_>>();
let generated_rule_dispatch = render_generated_rule_dispatch_with_rule_names(
&generated_rules,
&direct_generated_rule_calls,
&data.rule_names,
&inline_action_statements,
track_alt_numbers,
embedded_step_render,
portable_local_data.step_render(),
);
// A non-default policy must reach the interpreter through the emitted
// runtime options, so its literal forces the options-carrying call shape.
//
// A `hook`-disposed coordinate falls through to the configured policy when
// its hook is unimplemented (the documented `SemIR → hook → policy` chain),
// so it does NOT escalate the global policy: doing so would flip unrelated
// `assume-true` coordinates in the same grammar to fail-loud. Users select
// fail-loud for missing hooks with `--sem-unknown=error` /
// `--require-full-semantics`; under the default policy a declined hook keeps
// historical pass-through, per coordinate.
let unknown_policy_literal = match options.sem_unknown {
SemUnknownPolicy::AssumeTrue => None,
SemUnknownPolicy::AssumeFalse => Some("antlr4_runtime::UnknownSemanticPolicy::AssumeFalse"),
SemUnknownPolicy::Hook | SemUnknownPolicy::Error => {
Some("antlr4_runtime::UnknownSemanticPolicy::Error")
}
};
let parse_rule_fallback = render_parser_parse_rule_fallback(
track_alt_numbers,
&predicates,
&rule_args,
has_action_dispatch,
has_predicate_dispatch,
unknown_policy_literal,
);
let parser_semantics_function = render_parser_semantics_function(&predicates, data)?;
let typed_hook_adapter = render_typed_hook_adapter(
&type_name,
&parser_typed_hook_mappings(data, grammar_source, patterns)?,
);
let typed_parser_constructor = if typed_hook_adapter.is_empty() {
String::new()
} else {
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
format!(
"impl<L, T> {type_name}<L, {adapter_name}<T>>\nwhere\n L: TokenSource,\n T: {trait_name},\n{{\n pub fn with_typed_hooks(input: CommonTokenStream<L>, hooks: T) -> Self {{\n Self::with_hooks(input, {adapter_name}::new(hooks))\n }}\n}}\n"
)
};
// The adaptive-direct path runs `parse_atn_rule_adaptive_or_fallback`, which
// falls back through `parse_atn_rule` without the `ParserRuntimeOptions`
// emitted below. A non-default unknown-predicate policy only reaches the
// interpreter through those options, so it must not take that shortcut, or
// an untranslated predicate would silently pass instead of applying the
// configured fail/assume-false behavior.
let adaptive_direct_allowed = !has_action_dispatch
&& !track_alt_numbers
&& !has_predicate_dispatch
&& unknown_policy_literal.is_none();
let embedded_noop_states = BTreeSet::new();
let action_method = render_parser_action_method(
!action_states.is_empty() && !options.embedded,
if options.embedded {
&embedded_noop_states
} else {
&noop_action_states
},
);
let parse_convenience = render_parser_parse_convenience(&type_name);
let base_initialization = render_parser_base_initialization(unknown_policy_literal);
let public_rule_method_names = parser_public_rule_method_names(&data.rule_names);
let entry_rule_indices = likely_parser_entry_rule_indices(data)?;
let parser_rustdoc = render_parser_rustdoc(&public_rule_method_names, &entry_rule_indices);
let rule_methods = render_public_rule_methods(&public_rule_method_names);
let (
embedded_attrs_structs,
embedded_module_items,
embedded_struct_fields,
embedded_field_inits,
embedded_impl_items,
) = embedded_render_slots(embedded_data.as_ref());
let generated_header = GENERATED_MODULE_HEADER;
let generated_footer = GENERATED_MODULE_FOOTER;
let embedded_imports = if options.embedded {
"#[allow(unused_imports)]\nuse std::io::Write as _;\n#[allow(unused_imports)]\nuse antlr4_runtime::{{java_style_list, PredictionMode, BailErrorStrategy, TerminalNodeView as RuntimeTerminalNode, ErrorNodeView as RuntimeErrorNode, RuleNodeView, FromRuleNode, Token as _}};\n"
} else {
""
};
Ok(format!(
r#"{generated_header}use antlr4_runtime::recognizer::RecognizerData;
use antlr4_runtime::token::TokenSource;
use antlr4_runtime::token_stream::CommonTokenStream;
use antlr4_runtime::atn::parser_atn::ParserAtn;
use antlr4_runtime::{{BaseParser, GeneratedParser, GrammarMetadata, Parser, Recognizer}};
use std::sync::OnceLock;
{embedded_imports}
{token_constants}
{rule_constants}
{metadata}
{parser_semantics_function}
{typed_hook_adapter}
{embedded_attrs_structs}
{embedded_module_items}
static PARSER_ATN_DATA: &[u32] = &{parser_atn_data};
static ATN_CELL: OnceLock<ParserAtn> = OnceLock::new();
/// Validates and caches the packed grammar ATN for all parser instances.
fn atn() -> &'static ParserAtn {{
ATN_CELL.get_or_init(|| {{
ParserAtn::from_static(PARSER_ATN_DATA)
.unwrap_or_else(|error| panic!("generated parser ATN is incompatible with this runtime: {{error}}"))
}})
}}
/// Borrows the validated packed parser ATN embedded in this module.
pub fn parser_atn() -> &'static ParserAtn {{
atn()
}}
{parse_convenience}
{parser_rustdoc}#[derive(Debug)]
pub struct {type_name}<L, H = antlr4_runtime::NoSemanticHooks>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
base: BaseParser<L, H>,
simulator: Option<antlr4_runtime::ParserAtnSimulator<'static>>,
generated_only: bool,
{embedded_struct_fields}}}
#[allow(dead_code)]
#[derive(Debug)]
enum GeneratedRuleError {{
Fatal(antlr4_runtime::AntlrError),
}}
impl GeneratedRuleError {{
fn into_error(self) -> antlr4_runtime::AntlrError {{
match self {{
Self::Fatal(error) => error,
}}
}}
}}
impl<L> {type_name}<L, antlr4_runtime::NoSemanticHooks>
where
L: TokenSource,
{{
pub fn new(input: CommonTokenStream<L>) -> Self {{
Self::with_hooks(input, antlr4_runtime::NoSemanticHooks)
}}
}}
impl<L, H> {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
pub fn with_hooks(input: CommonTokenStream<L>, hooks: H) -> Self {{
let grammar_metadata = metadata();
let data = RecognizerData::new(
grammar_metadata.grammar_file_name(),
grammar_metadata.vocabulary(),
)
.with_rule_names(grammar_metadata.rule_names().iter().copied())
.with_channel_names(grammar_metadata.channel_names().iter().copied())
.with_mode_names(grammar_metadata.mode_names().iter().copied());
{base_initialization}
Self {{
base,
simulator: None,
generated_only: std::env::var_os("ANTLR4_RUST_GENERATED_ONLY").is_some(),
{embedded_field_inits} }}
}}
pub fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
/// Adds a listener for parser diagnostics.
pub fn add_error_listener<T>(&mut self, listener: T)
where
T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,
{{
self.base.add_error_listener(listener);
}}
/// Removes every parser error listener, including the default console listener.
pub fn remove_error_listeners(&mut self) {{
self.base.remove_error_listeners();
}}
/// Fully resets parser-owned state and rewinds the current token stream.
pub fn reset(&mut self) {{
self.base.reset();
if let Some(simulator) = self.simulator.as_mut() {{
simulator.reset();
}}
}}
/// Replaces the token stream and fully resets parser-owned state.
pub fn set_token_stream(&mut self, input: CommonTokenStream<L>) {{
self.base.set_token_stream(input);
if let Some(simulator) = self.simulator.as_mut() {{
simulator.reset();
}}
}}
#[must_use]
pub const fn token_stream(&self) -> &CommonTokenStream<L> {{
self.base.token_stream()
}}
#[must_use]
pub const fn token_stream_mut(&mut self) -> &mut CommonTokenStream<L> {{
self.base.token_stream_mut()
}}
#[must_use]
pub const fn token_store(&self) -> &antlr4_runtime::TokenStore {{
self.base.token_store()
}}
#[must_use]
pub const fn parse_tree_storage(&self) -> &antlr4_runtime::ParseTreeStorage {{
self.base.parse_tree_storage()
}}
#[must_use]
pub fn prediction_context_stats(&self) -> antlr4_runtime::PredictionContextStats {{
self.simulator.as_ref().map_or_else(
antlr4_runtime::PredictionContextStats::default,
antlr4_runtime::ParserAtnSimulator::prediction_context_stats,
)
}}
#[must_use]
pub fn parser_dfa_stats(&self) -> antlr4_runtime::ParserDfaStats {{
self.simulator.as_ref().map_or_else(
antlr4_runtime::ParserDfaStats::default,
antlr4_runtime::ParserAtnSimulator::parser_dfa_stats,
)
}}
/// Clears this grammar's learned parser decision DFAs.
pub fn clear_dfa(&mut self) {{
if let Some(simulator) = self.simulator.as_mut() {{
simulator.clear_dfa();
}} else {{
antlr4_runtime::ParserAtnSimulator::clear_shared_dfa(atn());
}}
}}
#[must_use]
pub fn node(&self, id: antlr4_runtime::NodeId) -> antlr4_runtime::Node<'_> {{
self.base.node(id)
}}
#[must_use]
pub fn into_token_stream(self) -> CommonTokenStream<L> {{
self.base.into_token_stream()
}}
#[must_use]
pub fn into_token_store(self) -> antlr4_runtime::TokenStore {{
self.base.into_token_store()
}}
#[must_use]
pub fn into_parsed_file(self, root: antlr4_runtime::NodeId) -> antlr4_runtime::ParsedFile {{
self.base.into_parsed_file(root)
}}
#[allow(dead_code)]
fn simulator(&mut self) -> &mut antlr4_runtime::ParserAtnSimulator<'static> {{
self.simulator
.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()))
}}
#[allow(dead_code)]
fn generated_only(&self) -> bool {{
self.generated_only
}}
#[allow(dead_code)]
fn parse_rule(&mut self, rule_index: usize) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_rule_precedence(rule_index, 0)
}}
#[allow(dead_code)]
fn parse_rule_precedence(&mut self, rule_index: usize, precedence: i32) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_rule_precedence_inner(rule_index, precedence, true)
}}
#[allow(dead_code)]
fn parse_rule_precedence_from_generated(&mut self, rule_index: usize, precedence: i32) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_rule_precedence_inner(rule_index, precedence, false)
}}
#[allow(dead_code)]
fn parse_rule_precedence_inner(&mut self, rule_index: usize, precedence: i32, allow_generated_fallback: bool) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
if allow_generated_fallback {{
// True top-level entry: drop any fail-loud coordinates left by a
// previous parse so a reused parser starts clean. Mid-parse the hits
// are preserved so a generated parent can surface a recovered child's
// fail-loud coordinate at this boundary.
self.base.reset_unknown_semantic_hits();
}}
let __rule_start = antlr4_runtime::IntStream::index(self.base.input());
let __generated_only = self.generated_only();
let __tree = if let Some(result) = self.parse_generated_rule(rule_index, precedence, allow_generated_fallback) {{
match result {{
Ok(tree) => tree,
Err(error) => {{
antlr4_runtime::IntStream::seek(self.base.input(), __rule_start);
// A generated predicate that consulted an unimplemented hook
// (returning None under the Error policy) fails the alternative
// and surfaces here as a generic failed-predicate/rule error.
// The documented contract is to fail loud with
// `AntlrError::Unsupported`, so prefer a recorded semantic error
// over the generic one — but only at the top-level entry, mirroring
// the post-parse check below: a nested child keeps its hits so the
// generated parent surfaces them at that boundary instead.
if allow_generated_fallback {{
if let Some(semantic_error) = self.base.take_unknown_semantic_error() {{
return Err(semantic_error);
}}
}}
return Err(error.into_error());
}}
}}
}} else if __generated_only {{
return Err(antlr4_runtime::AntlrError::Unsupported(format!("generated parser did not emit rule {{}}", rule_index)));
}} else {{
self.parse_interpreted_rule_precedence(rule_index, precedence)?
}};
// Surface unknown-predicate coordinates recorded under the Error policy
// at the top-level entry. Generated predicate steps evaluate on the
// committed path and are recovered as rule errors, so a parse that
// consulted an unimplemented hook predicate must fail with
// `AntlrError::Unsupported` instead of returning a recovered `Ok` tree.
if allow_generated_fallback {{
if let Some(error) = self.base.take_unknown_semantic_error() {{
return Err(error);
}}
}}
if allow_generated_fallback {{
self.base.report_generated_parser_diagnostics();
if let Some(error) = self.base.take_unknown_semantic_error() {{
return Err(error);
}}
}}
Ok(__tree)
}}
#[allow(dead_code)]
fn parse_interpreted_rule(&mut self, rule_index: usize) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
self.parse_interpreted_rule_precedence(rule_index, 0)
}}
#[allow(dead_code)]
fn parse_interpreted_rule_precedence(&mut self, rule_index: usize, precedence: i32) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError> {{
if precedence == 0 && {adaptive_direct_allowed} && std::env::var_os("ANTLR4_RUST_ADAPTIVE_DIRECT").is_some() {{
let simulator = self
.simulator
.get_or_insert_with(|| antlr4_runtime::ParserAtnSimulator::new_shared(atn()));
self.base
.parse_atn_rule_adaptive_or_fallback(atn(), simulator, rule_index)
}} else {{
{parse_rule_fallback}
}}
}}
{generated_rule_dispatch}
{embedded_impl_items}
{rule_methods}
{action_method}
}}
{typed_parser_constructor}
impl<L, H> GeneratedParser for {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
fn metadata() -> &'static GrammarMetadata {{
metadata()
}}
fn parser_atn() -> &'static ParserAtn {{
parser_atn()
}}
}}
impl<L, H> Recognizer for {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
fn data(&self) -> &antlr4_runtime::RecognizerData {{
self.base.data()
}}
fn data_mut(&mut self) -> &mut antlr4_runtime::RecognizerData {{
self.base.data_mut()
}}
}}
impl<L, H> Parser for {type_name}<L, H>
where
L: TokenSource,
H: antlr4_runtime::SemanticHooks,
{{
fn build_parse_trees(&self) -> bool {{ self.base.build_parse_trees() }}
fn set_build_parse_trees(&mut self, build: bool) {{ self.base.set_build_parse_trees(build); }}
fn number_of_syntax_errors(&self) -> usize {{ self.base.number_of_syntax_errors() }}
fn report_diagnostic_errors(&self) -> bool {{ self.base.report_diagnostic_errors() }}
fn set_report_diagnostic_errors(&mut self, report: bool) {{ self.base.set_report_diagnostic_errors(report); }}
fn prediction_mode(&self) -> antlr4_runtime::PredictionMode {{ self.base.prediction_mode() }}
fn set_prediction_mode(&mut self, mode: antlr4_runtime::PredictionMode) {{ self.base.set_prediction_mode(mode); }}
}}
{generated_footer}"#
))
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum ActionTemplate {
LexerPopMode,
Hook(SemanticHelperCall),
UnsupportedLexerAction { rule_name: String, body: String },
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum PredicateTemplate {
Hook,
True,
False,
FalseWithMessage {
message: String,
},
/// A non-constant-false predicate carrying an ANTLR `<fail=...>` message.
/// Transparent to evaluation — `inner` provides the truth value and codegen;
/// the message is surfaced only when `inner` returns false at runtime.
WithFailMessage {
inner: Box<Self>,
message: String,
},
Invoke {
value: bool,
},
LocalIntEquals {
value: i64,
},
LocalIntLessOrEqual {
value: i64,
},
LookaheadTextEquals {
offset: isize,
text: String,
},
TextEquals(String),
TokenStartColumnEquals(usize),
ColumnLessThan(usize),
ColumnGreaterOrEqual(usize),
LookaheadNotEquals {
offset: isize,
token_name: String,
},
TokenPairAdjacent,
ContextChildRuleTextNotEquals {
rule_name: String,
text: String,
},
}
fn can_generate_parser_predicate(predicate: &PredicateTemplate) -> bool {
// A `<fail=...>` wrapper is transparent: generatability follows the inner.
matches!(
predicate_effective_template(predicate),
PredicateTemplate::Hook
| PredicateTemplate::True
| PredicateTemplate::False
| PredicateTemplate::FalseWithMessage { .. }
| PredicateTemplate::Invoke { .. }
| PredicateTemplate::LocalIntEquals { .. }
| PredicateTemplate::LocalIntLessOrEqual { .. }
| PredicateTemplate::LookaheadTextEquals { .. }
| PredicateTemplate::LookaheadNotEquals { .. }
| PredicateTemplate::TokenPairAdjacent
| PredicateTemplate::ContextChildRuleTextNotEquals { .. }
)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RuleArgTemplate {
Literal(i64),
InheritLocal,
}
/// Translates the lexer recognizer surface used by rendered test bodies onto
/// the `BaseLexer` hooks: `self.text()` / column accessors become position
/// -aware `_base` calls, `self.output()` becomes stdout.
fn translate_embedded_lexer_body(body: &str, position_expr: &str) -> io::Result<String> {
let mut out = replace_all(body.trim(), "self.output()", "std::io::stdout()");
out = replace_all(
&out,
"self.text()",
&format!("_base.token_text_until({position_expr})"),
);
out = replace_all(
&out,
"self.char_position_in_line()",
&format!("_base.column_at({position_expr})"),
);
out = replace_all(
&out,
"self.token_start_char_position_in_line()",
"_base.token_start_column()",
);
if out.contains("self.") {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported recognizer surface in embedded lexer body: {body}"),
));
}
Ok(out)
}
/// Pairs verbatim lexer action bodies with serialized custom-action
/// coordinates, mirroring `lexer_action_templates`'s source walk.
fn embedded_lexer_actions(
data: &InterpData,
source: &str,
) -> io::Result<Vec<((i32, i32), String)>> {
let coordinates = lexer_custom_actions(data)?;
if coordinates.is_empty() {
return Ok(Vec::new());
}
let mut bodies = Vec::new();
let mut offset = 0;
while let Some(block) = next_action_block(source, offset) {
offset = block.after_brace;
if !is_lexer_custom_action_block(source, &block, &data.rule_names) {
continue;
}
bodies.push(translate_embedded_lexer_body(
block.body,
"action.position()",
)?);
}
if coordinates.len() != bodies.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"lexer ATN has {} custom action(s), but the rendered grammar yielded {} embedded bodies",
coordinates.len(),
bodies.len()
),
));
}
Ok(coordinates.into_iter().zip(bodies).collect())
}
/// Pairs verbatim lexer predicate bodies with serialized predicate
/// coordinates, mirroring `lexer_predicate_templates`'s source walk.
fn embedded_lexer_predicates(
data: &InterpData,
source: &str,
) -> io::Result<Vec<((usize, usize), String)>> {
let coordinates = lexer_predicate_transitions(data)?;
if coordinates.is_empty() {
return Ok(Vec::new());
}
let mut expressions = Vec::new();
let mut offset = 0;
while let Some(block) = next_predicate_action_block(source, offset) {
offset = block.after_brace;
// In a combined grammar, skip parser-rule predicates (they have no
// lexer coordinate). Lexer rule names start uppercase.
if !predicate_block_included_for_lexer(source, block.open_brace, &data.rule_names) {
continue;
}
expressions.push(translate_embedded_lexer_body(
block.body,
"predicate.position()",
)?);
}
if coordinates.len() != expressions.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"lexer ATN has {} predicate transition(s), but the rendered grammar yielded {} embedded bodies",
coordinates.len(),
expressions.len()
),
));
}
Ok(coordinates.into_iter().zip(expressions).collect())
}
/// `predicate_block_included` against the LEXER rule inventory.
fn predicate_block_included_for_lexer(
source: &str,
open_brace: usize,
rule_names: &[String],
) -> bool {
predicate_block_included(source, open_brace, rule_names)
}
fn render_embedded_lexer_action_method(actions: &[((i32, i32), String)]) -> String {
if actions
.iter()
.all(|(_, statement)| statement.trim().is_empty())
{
return String::new();
}
let mut arms = String::new();
for ((rule_index, action_index), statement) in actions {
writeln!(
arms,
" ({rule_index}, {action_index}) => {{ {statement} true }}"
)
.expect("writing to a string cannot fail");
}
arms.push_str(" _ => false,\n");
format!(
" fn run_action(_base: &mut BaseLexer<I>, action: antlr4_runtime::LexerCustomAction) -> bool {{\n #[allow(unused_imports)]\n use std::io::Write as _;\n match (action.rule_index(), action.action_index()) {{\n{arms} }}\n }}\n"
)
}
fn render_embedded_lexer_predicate_method(predicates: &[((usize, usize), String)]) -> String {
if predicates.is_empty() {
return String::new();
}
let mut arms = String::new();
for ((rule_index, pred_index), expression) in predicates {
writeln!(
arms,
" ({rule_index}, {pred_index}) => Some({{ {expression} }}),"
)
.expect("writing to a string cannot fail");
}
arms.push_str(" _ => None,\n");
format!(
" fn run_predicate(_base: &BaseLexer<I>, predicate: antlr4_runtime::LexerPredicate) -> Option<bool> {{\n match (predicate.rule_index(), predicate.pred_index()) {{\n{arms} }}\n }}\n"
)
}
/// Pairs supported lexer target-template actions with serialized custom-action
/// coordinates from the lexer ATN.
fn lexer_action_templates(
data: &InterpData,
grammar_source: &str,
allow_unsupported_only: bool,
patterns: &SemPatternFile,
) -> io::Result<Vec<((i32, i32), ActionTemplate)>> {
let actions = lexer_custom_actions(data)?;
if actions.is_empty() {
return Ok(Vec::new());
}
let templates =
extract_lexer_action_templates_with_patterns(grammar_source, &data.rule_names, patterns)?;
// With no source-side lexer action blocks there is nothing to align
// positionally. Leave every ATN coordinate uncovered so the selected
// semantic policy and semantics manifest account for them.
if templates.is_empty() {
return Ok(Vec::new());
}
if actions.len() == templates.len() {
reject_unsupported_lexer_action_templates(&templates, allow_unsupported_only)?;
return Ok(actions.into_iter().zip(templates).collect());
}
Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"lexer ATN has {} custom action(s), but grammar source yielded {} lexer action template(s)",
actions.len(),
templates.len()
),
))
}
/// Extracts lexer action templates in the same source order ANTLR uses for
/// custom lexer-action indexes.
#[cfg(test)]
fn extract_lexer_action_templates(
grammar_source: &str,
rule_names: &[String],
) -> Vec<ActionTemplate> {
extract_lexer_action_templates_with_patterns(
grammar_source,
rule_names,
&SemPatternFile::default(),
)
.expect("empty semantic patterns cannot fail")
}
fn extract_lexer_action_templates_with_patterns(
grammar_source: &str,
rule_names: &[String],
patterns: &SemPatternFile,
) -> io::Result<Vec<ActionTemplate>> {
let mut actions = Vec::new();
let mut offset = 0;
while let Some(block) = next_action_block(grammar_source, offset) {
offset = block.after_brace;
if !is_lexer_custom_action_block(grammar_source, &block, rule_names) {
continue;
}
let template = match parse_lexer_action_block_template(block.body) {
Some(template) => template,
None => patterns
.hook_helper_call(SemanticsKind::LexerAction, block.body)?
.map_or_else(
|| {
unsupported_lexer_action_template(
grammar_source,
block.open_brace,
block.body,
)
},
ActionTemplate::Hook,
),
};
actions.push(template);
}
Ok(actions)
}
/// Reports whether an action block found in grammar source participates in
/// ANTLR's lexer custom-action numbering. Shared by template extraction and
/// the semantics-manifest span collector so their walks cannot drift apart.
fn is_lexer_custom_action_block(
source: &str,
block: &templates::TemplateBlock<'_>,
rule_names: &[String],
) -> bool {
!block.predicate
&& rule_action_included(source, block.open_brace, Some(rule_names))
&& !is_after_action(source, block.open_brace)
&& !is_init_action(source, block.open_brace)
&& !is_definitions_action(source, block.open_brace)
&& !is_members_action(source, block.open_brace)
&& !is_options_block(source, block.open_brace)
}
fn reject_unsupported_lexer_action_templates(
actions: &[ActionTemplate],
allow_unsupported_only: bool,
) -> io::Result<()> {
if let Some(ActionTemplate::UnsupportedLexerAction { rule_name, body }) = actions
.iter()
.find(|action| matches!(action, ActionTemplate::UnsupportedLexerAction { .. }))
{
let has_supported_dispatch = actions.iter().any(lexer_action_template_needs_dispatch);
if allow_unsupported_only && !has_supported_dispatch {
return Ok(());
}
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"unsupported embedded lexer action in rule {rule_name}: {{{body}}}; \
rewrite target-specific actions as portable lexer commands where possible"
),
));
}
Ok(())
}
fn parse_lexer_action_block_template(body: &str) -> Option<ActionTemplate> {
parse_lexer_pop_mode_action(body)
}
fn parse_lexer_pop_mode_action(body: &str) -> Option<ActionTemplate> {
let body = body
.chars()
.filter(|ch| !ch.is_ascii_whitespace() && *ch != ';')
.collect::<String>();
matches!(
body.as_str(),
"popMode()"
| "this.popMode()"
| "if(!_modeStack.isEmpty()){popMode()}"
| "if(!this._modeStack.isEmpty()){popMode()}"
| "if(!_modeStack.isEmpty())popMode()"
| "if(!this._modeStack.isEmpty())popMode()"
)
.then_some(ActionTemplate::LexerPopMode)
}
fn unsupported_lexer_action_template(
source: &str,
open_brace: usize,
body: &str,
) -> ActionTemplate {
let rule = statement_rule_header(source, open_brace).map_or("<unknown>", |header| header.name);
ActionTemplate::UnsupportedLexerAction {
rule_name: rule.to_owned(),
body: one_line_action_body(body),
}
}
fn one_line_action_body(body: &str) -> String {
const ACTION_SUMMARY_LIMIT: usize = 96;
let mut out = String::new();
for (index, part) in body.split_whitespace().enumerate() {
if index > 0 {
out.push(' ');
}
out.push_str(part);
if out.len() > ACTION_SUMMARY_LIMIT {
let mut limit = ACTION_SUMMARY_LIMIT;
while !out.is_char_boundary(limit) {
limit -= 1;
}
out.truncate(limit);
out.push_str("...");
break;
}
}
out
}
fn rust_block_comment_text(value: &str) -> String {
let mut out = String::new();
let mut cursor = 0;
while let Some(relative_index) = value[cursor..].find("*/") {
let index = cursor + relative_index;
out.push_str(&value[cursor..index]);
out.push_str("* /");
cursor = index + 2;
}
if cursor == 0 {
value.to_owned()
} else {
out.push_str(&value[cursor..]);
out
}
}
/// Pairs supported lexer semantic predicates with serialized predicate
/// coordinates from the lexer ATN.
fn lexer_predicate_templates(
data: &InterpData,
grammar_source: &str,
patterns: &SemPatternFile,
) -> io::Result<Vec<((usize, usize), PredicateTemplate)>> {
let predicates = lexer_predicate_transitions(data)?;
if predicates.is_empty() {
return Ok(Vec::new());
}
let slots = extract_predicate_template_slots_filtered(
grammar_source,
patterns,
Some(&data.rule_names),
SemanticsKind::LexerPredicate,
)?;
if slots.iter().all(Option::is_none) {
return Ok(Vec::new());
}
// Each in-scope predicate block is one slot, aligned 1:1 with the lexer
// ATN's predicate transitions in source order. A mismatch means the scraper
// desynchronized from the ATN — a hard error. But a *translated* slot count
// below the transition count is fine: the untranslated coordinates simply
// stay uncovered and hit `run_predicate`'s policy catch-all.
if slots.len() != predicates.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"grammar has {} predicate block(s), but lexer ATN has {} predicate transition(s)",
slots.len(),
predicates.len()
),
));
}
// Apply per-coordinate `--sem-patterns` overrides before deciding a slot is
// unhonorable, mirroring the parser path: a `dispose = "assume-false"` /
// `"assume-true"` override on a lexer predicate coordinate replaces the
// body-derived slot (e.g. a helper that lowered to `Hook`) with the
// documented fallback, so the manifest's disposition is honored instead of
// failing generation. Slots are position-aligned 1:1 with `predicates`.
let mut slots = slots;
for (slot, (rule_index, pred_index)) in slots.iter_mut().zip(predicates.iter().copied()) {
if let Some(override_template) = patterns.coordinate_predicate_template(
SemanticsKind::LexerPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
) {
*slot = override_template;
}
}
// Pair only the translated slots with their coordinate; leave untranslated
// ones (None) uncovered so `render_lexer_predicate_method`'s catch-all
// applies the documented fallback for mixed lexers.
Ok(predicates
.into_iter()
.zip(slots)
.filter_map(|(coordinate, slot)| slot.map(|template| (coordinate, template)))
.collect())
}
/// Pairs supported parser semantic predicates with serialized predicate
/// coordinates from the parser ATN.
/// Builds parser predicate templates purely from `--sem-patterns`
/// per-coordinate overrides, keyed by ATN predicate coordinate.
///
/// Used when no grammar source is available: coordinate overrides resolve by
/// `(rule, index)` alone, so a `dispose = "hook" | "assume-true" |
/// "assume-false"` override still takes effect in the generated parser. (An
/// `error` override lowers to no template and is surfaced by
/// `enforce_sem_unknown` at codegen instead.)
fn parser_predicate_templates_from_overrides(
data: &InterpData,
patterns: &SemPatternFile,
) -> io::Result<Vec<((usize, usize), PredicateTemplate)>> {
let mut mapped = Vec::new();
for (rule_index, pred_index) in parser_predicate_transitions(data)? {
if let Some(Some(template)) = patterns.coordinate_predicate_template(
SemanticsKind::ParserPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
) {
mapped.push(((rule_index, pred_index), template));
}
}
Ok(mapped)
}
fn parser_predicate_templates(
data: &InterpData,
grammar_source: &str,
patterns: &SemPatternFile,
) -> io::Result<Vec<((usize, usize), PredicateTemplate)>> {
let predicates = parser_predicate_transitions(data)?;
let mut mapped = Vec::new();
let mut offset = 0;
let mut predicate_index = 0;
while let Some(block) = next_predicate_action_block(grammar_source, offset) {
offset = block.after_brace;
// In a combined grammar the scan also sees lexer-rule predicates, which
// have no parser ATN transition. Skip them without consuming a parser
// predicate index, so later parser predicates stay paired with the right
// coordinate instead of drifting (or erroring with "no parser ATN
// predicate transition"). Only skip blocks that positively resolve to a
// non-parser rule; an unresolvable header keeps the block.
if !predicate_block_included(grammar_source, block.open_brace, &data.rule_names) {
continue;
}
let coordinates = predicates.get(predicate_index).copied();
let override_template = coordinates.and_then(|(rule_index, pred_index)| {
patterns.coordinate_predicate_template(
SemanticsKind::ParserPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
)
});
let template = match override_template {
Some(template) => template,
None => parse_predicate_template_with_patterns_kind(
block.body,
patterns,
SemanticsKind::ParserPredicate,
)?,
};
if let Some(template) = template {
let template = match predicate_fail_message(grammar_source, block.after_brace) {
Some(message) => predicate_template_with_fail_message(template, message),
None => template,
};
let Some(coordinates) = coordinates else {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"grammar predicate template <{}> has no parser ATN predicate transition",
block.body
),
));
};
mapped.push((coordinates, template));
}
predicate_index += 1;
}
Ok(mapped)
}
/// Attaches ANTLR's `<fail=...>` option to a predicate template so the runtime
/// can surface the grammar-supplied message when the predicate fails at runtime.
///
/// A constant-false predicate folds into `FalseWithMessage` (its own dedicated
/// variant). Any *other* template — a hook, lookahead, member, or local-int
/// predicate that can also return false at runtime — is wrapped in
/// `WithFailMessage` so the message is preserved rather than discarded; the
/// wrapper is transparent to evaluation (see `predicate_effective_template`) and
/// only contributes the failure message.
fn predicate_template_with_fail_message(
template: PredicateTemplate,
message: String,
) -> PredicateTemplate {
match template {
PredicateTemplate::False => PredicateTemplate::FalseWithMessage { message },
// Already message-carrying: replace the message (a later `<fail=...>`
// wins) rather than nesting wrappers.
PredicateTemplate::FalseWithMessage { .. } => {
PredicateTemplate::FalseWithMessage { message }
}
PredicateTemplate::WithFailMessage { inner, .. } => {
PredicateTemplate::WithFailMessage { inner, message }
}
other => PredicateTemplate::WithFailMessage {
inner: Box::new(other),
message,
},
}
}
/// The evaluation-relevant template, unwrapping a `WithFailMessage` wrapper.
/// The wrapper only carries a failure message; its runtime truth value and
/// codegen come from the inner template.
fn predicate_effective_template(template: &PredicateTemplate) -> &PredicateTemplate {
match template {
PredicateTemplate::WithFailMessage { inner, .. } => inner,
other => other,
}
}
/// The grammar-supplied `<fail=...>` message a template carries, if any.
fn predicate_template_fail_message(template: &PredicateTemplate) -> Option<&str> {
match template {
PredicateTemplate::FalseWithMessage { message }
| PredicateTemplate::WithFailMessage { message, .. } => Some(message),
_ => None,
}
}
/// Computes the position-to-state offset ANTLR's action ordering implies.
///
/// Source-span walks and the serialized ATN action states can differ in count:
/// ANTLR may synthesize action states ahead of authored action blocks. This
/// resolves the leading offset that aligns walked span `i` with
/// `states[offset + i]`.
fn action_slot_state_offset(states_len: usize, slot_len: usize) -> io::Result<usize> {
if states_len > slot_len {
// More ATN action states than resolved source slots: the extra states
// are synthetic actions ANTLR inserted at the rule head (before the
// author's), so skip that many leading states to align source slot `i`
// with the author action it describes.
return Ok(states_len - slot_len);
}
if states_len != slot_len {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"grammar has {slot_len} parser action source slot(s), but parser ATN has {states_len} action transition(s)"
),
));
}
Ok(0)
}
/// Pairs action-block source slots with ATN action states using per-rule
/// offsets. ANTLR can synthesize action states independently in each rule, and
/// left-recursion rewriting serializes primary alternatives before operator
/// alternatives, so a global source-order offset is not stable.
fn assign_states_to_parser_action_slots(
data: &InterpData,
source: &str,
slots: Vec<ParserActionSourceSlot>,
) -> io::Result<Vec<(usize, ParserActionSourceSlot)>> {
let model = embedded::parse_embedded_rules_model(source, &data.rule_names);
assign_states_to_parser_action_slots_with_model(data, &model, slots)
}
fn assign_states_to_parser_action_slots_with_model(
data: &InterpData,
model: &embedded::EmbeddedModel,
slots: Vec<ParserActionSourceSlot>,
) -> io::Result<Vec<(usize, ParserActionSourceSlot)>> {
if slots.is_empty() {
return Ok(Vec::new());
}
let action_state_rules = parser_action_state_rules(data)?;
let mut slots_by_rule: BTreeMap<usize, Vec<ParserActionSourceSlot>> = BTreeMap::new();
for slot in slots {
slots_by_rule.entry(slot.rule_index).or_default().push(slot);
}
let mut states_by_rule: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
for state in parser_action_states(data)? {
let Some(rule_index) = action_state_rules.get(&state).copied() else {
continue;
};
states_by_rule.entry(rule_index).or_default().push(state);
}
let mut assigned = Vec::new();
for (rule_index, mut rule_slots) in slots_by_rule {
if let Some(rule) = model.rules.get(rule_index) {
let is_left_recursive = rule.alts.iter().any(|alt| alt.is_lr_operator(&rule.name));
if is_left_recursive {
let is_operator_slot = |offset: usize| {
rule.alts
.iter()
.find(|alt| alt.span.0 <= offset && offset < alt.span.1)
.is_some_and(|alt| alt.is_lr_operator(&rule.name))
};
let (primary, operators): (Vec<_>, Vec<_>) = rule_slots
.into_iter()
.partition(|slot| !is_operator_slot(slot.source_offset));
rule_slots = primary.into_iter().chain(operators).collect();
}
}
let states = states_by_rule.remove(&rule_index).unwrap_or_default();
let state_offset =
action_slot_state_offset(states.len(), rule_slots.len()).map_err(|error| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("rule {rule_index}: {error}"),
)
})?;
for (index, slot) in rule_slots.into_iter().enumerate() {
let Some(state) = states.get(state_offset + index).copied() else {
continue;
};
assigned.push((state, slot));
}
}
assigned.sort_by_key(|(state, _)| *state);
Ok(assigned)
}
/// Applies an optional rule-name filter to an action or signature position.
fn rule_action_included(source: &str, position: usize, rule_names: Option<&[String]>) -> bool {
let Some(header) = statement_rule_header(source, position) else {
return rule_names.is_none();
};
rule_names.is_none_or(|names| names.iter().any(|name| name == header.name))
&& !has_prior_rule_definition(source, header.name, header.start)
}
fn is_parser_rule_body_action_block(
source: &str,
block: &templates::TemplateBlock<'_>,
rule_names: Option<&[String]>,
) -> bool {
rule_action_included(source, block.open_brace, rule_names)
&& !block.predicate
&& !is_after_action(source, block.open_brace)
&& !is_init_action(source, block.open_brace)
&& !is_definitions_action(source, block.open_brace)
&& !is_members_action(source, block.open_brace)
&& !is_options_block(source, block.open_brace)
&& !is_tokens_or_channels_block(source, block.open_brace)
&& !is_rule_exception_handler_block(source, block.open_brace)
}
fn is_tokens_or_channels_block(source: &str, open_brace: usize) -> bool {
let statement_start = statement_start_before(source, open_brace);
matches!(
source[statement_start..open_brace].trim(),
"tokens" | "channels"
)
}
fn is_rule_exception_handler_block(source: &str, open_brace: usize) -> bool {
let statement_start = statement_start_before(source, open_brace);
if last_rule_header_colon(source, open_brace).is_some_and(|colon| colon >= statement_start) {
return false;
}
let prefix = &source[statement_start..open_brace];
match rule_header_start_identifier(prefix) {
Some("finally") => true,
Some("catch") => prefix.contains('[') && prefix.contains(']'),
_ => false,
}
}
/// Reports whether a predicate block at `position` belongs to a rule this ATN
/// covers, tolerating headers this scraper cannot resolve.
///
/// Unlike [`rule_action_included`], a predicate block is *excluded only when its
/// owning rule name positively resolves and is absent from `rule_names`* — i.e.
/// it demonstrably belongs to the other rule set of a combined grammar. When
/// the header cannot be resolved (the brace-counting scraper is defeated by,
/// e.g., a `;` inside a comment above the rule) the block is kept, preserving
/// the pre-filter behavior rather than dropping a real predicate coordinate.
fn predicate_block_included(source: &str, position: usize, rule_names: &[String]) -> bool {
statement_rule_header(source, position)
.is_none_or(|header| rule_names.iter().any(|name| name == header.name))
}
fn next_action_block(source: &str, offset: usize) -> Option<templates::TemplateBlock<'_>> {
let open_brace = find_action_open_brace(source, offset)?;
let close_brace = matching_action_brace(source, open_brace + 1)?;
let after_brace = close_brace + 1;
Some(templates::TemplateBlock {
open_brace,
body: &source[open_brace + 1..close_brace],
after_brace,
predicate: source[after_brace..].trim_start().starts_with('?'),
})
}
fn find_action_open_brace(source: &str, offset: usize) -> Option<usize> {
templates::find_significant_open_brace(source, offset)
}
/// Finds grammar predicate templates in the same order as ANTLR serializes
/// predicate transitions.
#[cfg(test)]
fn extract_supported_predicate_templates(
grammar_source: &str,
patterns: &SemPatternFile,
) -> io::Result<Vec<PredicateTemplate>> {
extract_supported_predicate_templates_filtered(grammar_source, patterns, None)
}
#[cfg(test)]
fn extract_supported_predicate_templates_filtered(
grammar_source: &str,
patterns: &SemPatternFile,
rule_names: Option<&[String]>,
) -> io::Result<Vec<PredicateTemplate>> {
Ok(extract_predicate_template_slots_filtered(
grammar_source,
patterns,
rule_names,
SemanticsKind::LexerPredicate,
)?
.into_iter()
.flatten()
.collect())
}
/// Extracts one slot per in-scope predicate block, preserving position: a
/// translated block yields `Some(template)`, an untranslated (native) block
/// yields `None` (its coordinate falls through to the policy / catch-all). An
/// untranslated ANTLR `<...>` `StringTemplate` is still a hard codegen error.
fn extract_predicate_template_slots_filtered(
grammar_source: &str,
patterns: &SemPatternFile,
rule_names: Option<&[String]>,
kind: SemanticsKind,
) -> io::Result<Vec<Option<PredicateTemplate>>> {
let mut slots = Vec::new();
let mut offset = 0;
while let Some(block) = next_predicate_action_block(grammar_source, offset) {
offset = block.after_brace;
// In a combined grammar, skip predicate blocks that belong to a
// different rule set (e.g. parser-rule predicates while scanning the
// lexer), so positional pairing with this ATN's predicate transitions
// does not drift. Only skip blocks that positively resolve to a
// non-target rule; unresolvable headers keep the block.
let excluded = rule_names.is_some_and(|names| {
!predicate_block_included(grammar_source, block.open_brace, names)
});
if excluded {
continue;
}
if let Some(template) =
parse_predicate_template_with_patterns_kind(block.body, patterns, kind)?
{
slots.push(Some(template));
} else if is_unsupported_string_template_body(block.body) {
// An untranslated ANTLR `<...>` StringTemplate predicate is a
// codegen error (we can't render it). A native target-language
// comparison like `{this.level < 2}?` merely *contains* `<`; it is
// not a template and must fall through to the unknown-predicate
// policy (inventoried by `collect_parser_semantics`) instead of
// aborting generation under the documented fallbacks.
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported target predicate template <{}>", block.body),
));
} else {
// Native predicate the translator does not cover: keep the slot so
// positional pairing with ATN transitions holds; the coordinate
// stays uncovered and the `run_predicate` catch-all / policy applies.
slots.push(None);
}
}
Ok(slots)
}
/// Reports whether a predicate body is an untranslated ANTLR `<...>`
/// `StringTemplate` (a single template wrapper or a sequence of them), as
/// opposed to a native target-language predicate that merely contains a `<`
/// operator.
fn is_unsupported_string_template_body(body: &str) -> bool {
single_template_body(body).is_some() || template_sequence_bodies(body).is_some()
}
/// Parses an ANTLR semantic-predicate fail option following the predicate `?`.
fn predicate_fail_message(source: &str, after_brace: usize) -> Option<String> {
let rest = source[after_brace..].trim_start();
let rest = rest.strip_prefix('?')?.trim_start();
let rest = rest.strip_prefix("<fail=")?.trim_start();
let quote = rest.chars().next()?;
if quote != '\'' && quote != '"' {
return None;
}
let body_start = quote.len_utf8();
let body_end = rest[body_start..].find(quote)? + body_start;
let after_quote = body_end + quote.len_utf8();
if !rest[after_quote..].trim_start().starts_with('>') {
return None;
}
Some(rest[body_start..body_end].to_owned())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RuleHeader<'a> {
name: &'a str,
start: usize,
}
/// Returns the grammar rule that owns an action or signature position by reading
/// the current rule header before the first colon in the statement.
fn statement_rule_header(source: &str, position: usize) -> Option<RuleHeader<'_>> {
source.get(..position)?;
let colon = last_rule_header_colon(source, position)?;
let start = rule_statement_start(source, colon);
let header = &source[start..colon];
// The rule name is the first identifier in the header, after skipping
// leading comments, `<...>` templates, and grammar-level `@name {...}`
// blocks. Taking the *first* identifier keeps `continue returns [int x] :`
// resolving to `continue` (not the `returns` keyword or a `[...]` type),
// while the skips handle a preceding `@members {...}` / doc comment.
let name = rule_header_start_identifier(header)?;
Some(RuleHeader { name, start })
}
/// The byte offset just after the previous statement terminator (`;`) before
/// `colon`, i.e. the start of the current rule header.
///
/// Only a **top-level** `;` terminates the previous statement. A `;` inside a
/// braced action body (`@members { int x; }`, `{ native(); }`) or a `}` closing
/// such a block must not be treated as a boundary — otherwise the header start
/// lands mid-action and the rule name is lost. The scan skips balanced `{...}`
/// regions (and, via the grammar cursor, comments and string literals).
/// Grammar-level `@name {...}` blocks that precede the rule are elided by the
/// rule-name reader.
fn rule_statement_start(source: &str, colon: usize) -> usize {
statement_start_before(source, colon)
}
fn statement_start_before(source: &str, position: usize) -> usize {
let mut cursor = templates::GrammarSourceCursor::new(source, 0);
let mut start = 0;
while let Some((index, ch)) = cursor.next_significant() {
if index >= position {
break;
}
match ch {
'{' => {
// Skip the balanced action body so its inner `;`/`}` are ignored.
let resume = matching_action_brace(source, index + 1)
.map_or(position, |close| close.saturating_add(1).min(position));
cursor.seek(resume);
}
';' => start = index + 1,
_ => {}
}
}
start
}
fn last_rule_header_colon(source: &str, position: usize) -> Option<usize> {
let mut last = None;
let mut cursor = templates::GrammarSourceCursor::new(source, 0);
while let Some((index, ch)) = cursor.next_significant() {
if index >= position {
break;
}
match ch {
// Embedded action bodies may contain colons (Rust paths, ternary
// templates); skip the balanced block instead of scanning it.
'{' => cursor.seek(matching_action_brace(source, index + 1).map_or_else(
|| index + ch.len_utf8(),
|close| close.saturating_add(1).min(position),
)),
':' => last = Some(index),
_ => {}
}
}
last
}
/// Reports whether an earlier rule with the same name already owns the active
/// definition, matching ANTLR's import override rules for composite grammars.
fn has_prior_rule_definition(source: &str, name: &str, before: usize) -> bool {
let mut offset = 0;
while let Some(colon) = source[offset..before].find(':').map(|index| offset + index) {
let header_start = rule_statement_start(source, colon);
if rule_header_start_identifier(&source[header_start..colon]) == Some(name) {
return true;
}
offset = colon + 1;
}
false
}
fn uses_alt_number_contexts(source: &str) -> bool {
source.contains("<TreeNodeWithAltNumField") || source.contains("contextSuperClass")
}
/// Reads the rule name at the *start* of a rule header, i.e. the first
/// identifier after skipping leading `@name {...}` clauses, comments, and
/// `<...>` templates, and an optional `fragment` keyword. Stops before rule
/// option keywords (`returns`/`locals`/`throws`/`options`) and their `[...]` /
/// `{...}` payloads, so `continue returns [int x] :` still resolves to
/// `continue`.
fn rule_header_start_identifier(header: &str) -> Option<&str> {
let name = rule_header_identifier(header, true)?;
// A lexer rule may be introduced by `fragment`; the real name follows it.
if name == "fragment" {
let rest = &header[header.find("fragment")? + "fragment".len()..];
return rule_header_identifier(rest, true);
}
Some(name)
}
/// Shared rule-header identifier scan. With `first`, returns the first bareword
/// identifier (the rule name at a statement start); otherwise the last (used
/// when the caller has already sliced off everything after the rule name, e.g.
/// the text before an `@init`/`@after` marker). Skips comments, `@name {...}`
/// clauses, and `<...>` templates in both modes.
fn rule_header_identifier(header: &str, first: bool) -> Option<&str> {
let bytes = header.as_bytes();
let mut index = 0;
let mut found: Option<(usize, usize)> = None;
while index < bytes.len() {
// Skip comments so their words are not mistaken for the rule name.
if bytes[index] == b'/' && bytes.get(index + 1) == Some(&b'/') {
index += 2;
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
continue;
}
if bytes[index] == b'/' && bytes.get(index + 1) == Some(&b'*') {
index += 2;
while index < bytes.len()
&& !(bytes[index] == b'*' && bytes.get(index + 1) == Some(&b'/'))
{
index += 1;
}
index = (index + 2).min(bytes.len());
continue;
}
match bytes[index] {
// Skip a `@name {...}` block, including its (possibly brace-nested)
// body, so its internal identifiers are not mistaken for the rule.
b'@' => {
let mut cursor = index + 1;
while cursor < bytes.len() && !matches!(bytes[cursor], b'{' | b';' | b'\n') {
cursor += 1;
}
if cursor < bytes.len() && bytes[cursor] == b'{' {
let mut depth = 1_usize;
cursor += 1;
while cursor < bytes.len() && depth > 0 {
match bytes[cursor] {
b'{' => depth += 1,
b'}' => depth -= 1,
_ => {}
}
cursor += 1;
}
}
index = cursor;
}
// Skip a `<...>` template (a listener/member directive line), which is
// not a rule name.
b'<' => {
while index < bytes.len() && bytes[index] != b'>' {
index += 1;
}
index += usize::from(index < bytes.len());
}
byte if byte == b'_' || byte.is_ascii_alphanumeric() => {
let start = index;
while index < bytes.len()
&& (bytes[index] == b'_' || bytes[index].is_ascii_alphanumeric())
{
index += 1;
}
// A grammar-level declaration block (`tokens { ... }`,
// `options { ... }`, `channels { ... }`) is a bareword directly
// followed by `{`. It precedes the first rule and is not the rule
// name, so skip the keyword and its block and keep scanning for
// the actual name. (A rule name is followed by `:`, `returns`,
// `@`, `[`, etc. — never directly by `{`.)
let after_word = templates::skip_ascii_whitespace(header, index);
if first && header.as_bytes().get(after_word) == Some(&b'{') {
let mut depth = 1_usize;
let mut cursor = after_word + 1;
while cursor < bytes.len() && depth > 0 {
match bytes[cursor] {
b'{' => depth += 1,
b'}' => depth -= 1,
_ => {}
}
cursor += 1;
}
index = cursor;
continue;
}
found = Some((start, index));
if first {
break;
}
}
_ => index += 1,
}
}
found.map(|(start, end)| &header[start..end])
}
fn parse_predicate_template(body: &str) -> Option<PredicateTemplate> {
let body = body.trim();
if let Some(inner) = single_template_body(body) {
return parse_predicate_template(inner);
}
match body {
"True()" => Some(PredicateTemplate::True),
"False()" => Some(PredicateTemplate::False),
r#"ParserPropertyCall({$parser}, "Property()")"# => Some(PredicateTemplate::True),
_ => parse_raw_boolean_predicate(body)
.or_else(|| parse_text_equals_predicate(body))
.or_else(|| parse_token_start_column_equals_predicate(body))
.or_else(|| parse_column_compare_predicate(body))
.or_else(|| parse_invoke_predicate(body))
.or_else(|| parse_val_equals_predicate(body))
.or_else(|| parse_raw_local_int_less_or_equal_predicate(body))
.or_else(|| parse_csharp_parser_predicate(body))
.or_else(|| parse_lt_equals_predicate(body))
.or_else(|| parse_la_not_equals_predicate(body)),
}
}
fn parse_predicate_template_with_patterns(
body: &str,
patterns: &SemPatternFile,
) -> io::Result<Option<PredicateTemplate>> {
parse_predicate_template_with_patterns_kind(body, patterns, SemanticsKind::ParserPredicate)
}
fn parse_predicate_template_with_patterns_kind(
body: &str,
patterns: &SemPatternFile,
kind: SemanticsKind,
) -> io::Result<Option<PredicateTemplate>> {
Ok(match parse_predicate_template(body) {
Some(template) => Some(template),
None => patterns.predicate_template(kind, body)?,
})
}
fn parse_raw_boolean_predicate(body: &str) -> Option<PredicateTemplate> {
match body {
"true" => return Some(PredicateTemplate::True),
"false" => return Some(PredicateTemplate::False),
_ => {}
}
let (equals, left, right) = if let Some((left, right)) = body.split_once("==") {
(true, left, right)
} else {
let (left, right) = body.split_once("!=")?;
(false, left, right)
};
let left = left.trim().parse::<i64>().ok()?;
let right = right.trim().parse::<i64>().ok()?;
let value = if equals { left == right } else { left != right };
Some(if value {
PredicateTemplate::True
} else {
PredicateTemplate::False
})
}
/// Returns the call body for an action made of exactly one target template.
fn single_template_body(body: &str) -> Option<&str> {
let body = body.trim();
if body.as_bytes().first() != Some(&b'<') {
return None;
}
let close = matching_template_close(body, 1)?;
(close + 1 == body.len()).then_some(&body[1..close])
}
/// Parses simple local integer argument predicates such as
/// `ValEquals("$i","2")`.
fn parse_val_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let arguments = body
.strip_prefix("ValEquals(")
.and_then(|value| value.strip_suffix(')'))
.map(split_template_arguments)?;
let [local, value] = arguments.as_slice() else {
return None;
};
if parse_template_string(local)? != "$i" {
return None;
}
Some(PredicateTemplate::LocalIntEquals {
value: parse_template_string(value)?.parse::<i64>().ok()?,
})
}
/// Parses raw ANTLR semantic predicates such as `5 >= $_p`.
///
/// The Java generator lowers these against the generated context field
/// `_localctx._p`. The metadata runtime does not execute target code, so the
/// generator records the literal bound and the rule-call argument table makes
/// the current `_p` value available while interpreting the predicate
/// transition.
fn parse_raw_local_int_less_or_equal_predicate(body: &str) -> Option<PredicateTemplate> {
let (value, local) = body.split_once(">=")?;
if local.trim() != "$_p" {
return None;
}
Some(PredicateTemplate::LocalIntLessOrEqual {
value: value.trim().parse::<i64>().ok()?,
})
}
/// Parses the runtime-testsuite helper that prints when a predicate is
/// evaluated before returning the wrapped boolean value.
fn parse_invoke_predicate(body: &str) -> Option<PredicateTemplate> {
let value = body.strip_suffix(":Invoke_pred()")?;
match value {
"True()" => Some(PredicateTemplate::Invoke { value: true }),
"False()" => Some(PredicateTemplate::Invoke { value: false }),
r#"ValEquals("$i","99")"# => Some(PredicateTemplate::Invoke { value: true }),
_ => None,
}
}
fn parse_csharp_parser_predicate(body: &str) -> Option<PredicateTemplate> {
match body.trim() {
"this.IsRightArrow()" | "this.IsRightShift()" | "this.IsRightShiftAssignment()" => {
Some(PredicateTemplate::TokenPairAdjacent)
}
"this.IsLocalVariableDeclaration()" => {
Some(PredicateTemplate::ContextChildRuleTextNotEquals {
rule_name: "local_variable_type".to_owned(),
text: "var".to_owned(),
})
}
_ => None,
}
}
fn parse_text_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let argument = body
.strip_prefix("TextEquals(")
.and_then(|value| value.strip_suffix(')'))?;
Some(PredicateTemplate::TextEquals(parse_template_string(
argument,
)?))
}
fn parse_token_start_column_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let argument = body
.strip_prefix("TokenStartColumnEquals(")
.and_then(|value| value.strip_suffix(')'))?;
Some(PredicateTemplate::TokenStartColumnEquals(
parse_template_string(argument)?.parse().ok()?,
))
}
/// Parses lexer column predicates serialized by upstream templates as
/// `<Column()> \< 2` or `<Column()> >= 2`.
fn parse_column_compare_predicate(body: &str) -> Option<PredicateTemplate> {
let rest = body
.trim()
.strip_prefix("<Column()>")
.or_else(|| body.trim().strip_prefix("Column()"))?
.trim_start();
let rest = rest.strip_prefix('\\').unwrap_or(rest).trim_start();
if let Some(value) = rest.strip_prefix('<') {
return Some(PredicateTemplate::ColumnLessThan(
value.trim().parse().ok()?,
));
}
Some(PredicateTemplate::ColumnGreaterOrEqual(
rest.strip_prefix(">=")?.trim().parse().ok()?,
))
}
fn parse_la_not_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let arguments = body
.strip_prefix("LANotEquals(")
.and_then(|value| value.strip_suffix(')'))
.map(split_template_arguments)?;
let [offset, token] = arguments.as_slice() else {
return None;
};
let offset = parse_template_string(offset)?.parse::<isize>().ok()?;
let token_name = parse_parser_token_argument(token)?;
Some(PredicateTemplate::LookaheadNotEquals { offset, token_name })
}
/// Parses `LTEquals` predicates that compare lookahead token text.
///
/// The runtime-testsuite passes the expected text as a quoted target-language
/// string literal, so the decoded `StringTemplate` argument may still contain
/// one nested quote pair.
fn parse_lt_equals_predicate(body: &str) -> Option<PredicateTemplate> {
let arguments = body
.strip_prefix("LTEquals(")
.and_then(|value| value.strip_suffix(')'))
.map(split_template_arguments)?;
let [offset, text] = arguments.as_slice() else {
return None;
};
let offset = parse_template_string(offset)?.parse::<isize>().ok()?;
let text = parse_template_string(text)?;
let text = text
.strip_prefix('"')
.and_then(|value| value.strip_suffix('"'))
.unwrap_or(&text)
.to_owned();
Some(PredicateTemplate::LookaheadTextEquals { offset, text })
}
fn parse_parser_token_argument(argument: &str) -> Option<String> {
let body = argument
.trim()
.strip_prefix("{T<ParserToken(")?
.strip_suffix(")>}")?;
let parts = split_template_arguments(body);
let [_, token_name] = parts.as_slice() else {
return None;
};
parse_template_string(token_name)
}
/// Reads the lexer ATN to locate serialized custom action coordinates.
fn lexer_custom_actions(data: &InterpData) -> io::Result<Vec<(i32, i32)>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
Ok(atn
.lexer_actions()
.iter()
.filter_map(|action| match action {
LexerAction::Custom {
rule_index,
action_index,
} => Some((*rule_index, *action_index)),
_ => None,
})
.collect())
}
/// Reads the lexer ATN to locate semantic predicate coordinates.
fn lexer_predicate_transitions(data: &InterpData) -> io::Result<Vec<(usize, usize)>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut predicates = Vec::new();
for state in atn.states() {
for transition in &state.transitions {
if let LexerTransition::Predicate {
rule_index,
pred_index,
..
} = transition
{
predicates.push((*rule_index, *pred_index));
}
}
}
Ok(predicates)
}
/// Reads the packed parser ATN to locate semantic predicate coordinates.
fn parser_predicate_transitions(data: &InterpData) -> io::Result<Vec<(usize, usize)>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut predicates = Vec::new();
for state in atn.states() {
for transition in state.transitions() {
if let ParserTransitionData::Predicate {
rule_index,
pred_index,
..
} = transition.data()
{
predicates.push((rule_index, pred_index));
}
}
}
Ok(predicates)
}
/// Reads the parser ATN to locate action-transition source states.
fn parser_action_states(data: &InterpData) -> io::Result<Vec<usize>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut states = Vec::new();
for state in atn.states() {
if state
.transitions()
.iter()
.any(|transition| matches!(transition.data(), ParserTransitionData::Action { .. }))
{
states.push(state.state_number());
}
}
Ok(states)
}
/// Reads the parser ATN action transitions keyed by source state.
fn parser_action_state_rules(data: &InterpData) -> io::Result<BTreeMap<usize, usize>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut states = BTreeMap::new();
for state in atn.states() {
for transition in state.transitions() {
if let ParserTransitionData::Action { rule_index, .. } = transition.data() {
states.insert(state.state_number(), rule_index);
}
}
}
Ok(states)
}
/// Counts the author-written action `{...}` blocks per parser rule index.
///
/// Uses the general block walk (which surfaces *native*/untranslated blocks too,
/// unlike the translation-oriented `parser_action_source_block_slots`), so it
/// counts every action the grammar author actually wrote — including code we do
/// not translate. Predicates, `@init`/`@after`/members/definitions/options
/// blocks, and blocks in non-parser rules are excluded, matching what ANTLR
/// turns into a numbered rule-body action transition.
fn authored_parser_action_blocks_per_rule(
grammar_source: &str,
rule_names: &[String],
) -> BTreeMap<usize, usize> {
let mut counts = BTreeMap::new();
let mut offset = 0;
while let Some(block) = next_action_block(grammar_source, offset) {
offset = block.after_brace;
if !is_parser_rule_body_action_block(grammar_source, &block, Some(rule_names)) {
continue;
}
let Some(header) = statement_rule_header(grammar_source, block.open_brace) else {
continue;
};
if let Some(rule_index) = rule_names.iter().position(|name| name == header.name) {
*counts.entry(rule_index).or_insert(0) += 1;
}
}
counts
}
/// The parser ATN action states that ANTLR *synthesized* (during left-recursion
/// elimination and similar rewrites) rather than the author writing them.
///
/// A synthetic action has the same ATN shape and manifest disposition
/// (`Ignored`, no source body) as an author-written action we could not
/// translate, so they cannot be told apart from the ATN alone. The discriminator
/// is grammar-source correlation:
///
/// 1. An action state that a `{...}` source block was *attributed to* (via the
/// same span walk the manifest uses) is authored. Empty `{}` blocks are
/// handled separately as explicit no-op states, not ANTLR-synthetic states.
/// 2. A state with no attributed span is either ANTLR-synthetic or a *native*
/// authored action the translatable span walk did not surface. Distinguish by
/// count: per rule, the author wrote `authored_blocks(rule)` action blocks
/// total; `spanned(rule)` of them got a span, so `authored_blocks - spanned`
/// are native-authored and the *rest* of the span-less states are synthetic.
/// The span-less states are taken in ascending state order, native-authored
/// first (they still fail loud under `error`), synthetic after.
///
/// Ordering within the span-less group does not matter for correctness: a rule
/// mixes at most one of native-authored / synthetic in practice, and the count
/// split is exact. This is robust to ANTLR inserting its synthetic action at the
/// rule entry (a low state number) *before* the author's alternative actions —
/// the earlier positional "first N are authored" heuristic was not.
fn synthetic_parser_action_states(
data: &InterpData,
grammar_source: Option<&str>,
) -> io::Result<BTreeSet<usize>> {
let Some(grammar_source) = grammar_source else {
// Without grammar source we cannot correlate; treat nothing as synthetic
// so the manifest-only path keeps its existing (conservative) behavior.
return Ok(BTreeSet::new());
};
let authored = authored_parser_action_blocks_per_rule(grammar_source, &data.rule_names);
// States that a source block was attributed to (authored, incl. empty `{}`).
let spanned = assign_states_to_parser_action_slots(
data,
grammar_source,
parser_action_source_block_slots(grammar_source, &data.rule_names),
)?
.into_iter()
.map(|(state, _)| state)
.collect::<BTreeSet<usize>>();
// Per rule, how many authored blocks got a span. The remaining authored
// blocks are native (no span) and must NOT be exempted.
let mut spanned_per_rule = BTreeMap::new();
let state_rules = parser_action_state_rules(data)?;
for (state, rule_index) in &state_rules {
if spanned.contains(state) {
*spanned_per_rule.entry(*rule_index).or_insert(0_usize) += 1;
}
}
// Walk span-less states in state order; the first `native_authored` per rule
// are author-written natives (kept fail-loud), the rest are synthetic.
let mut native_seen = BTreeMap::new();
let mut synthetic = BTreeSet::new();
for (state, rule_index) in state_rules {
if spanned.contains(&state) {
continue;
}
let authored_in_rule = authored.get(&rule_index).copied().unwrap_or(0);
let spanned_in_rule = spanned_per_rule.get(&rule_index).copied().unwrap_or(0);
let native_authored = authored_in_rule.saturating_sub(spanned_in_rule);
let seen = native_seen.entry(rule_index).or_insert(0_usize);
if *seen < native_authored {
// An author-written native action with no span: keep it fail-loud.
*seen += 1;
} else {
synthetic.insert(state);
}
}
Ok(synthetic)
}
fn empty_parser_action_states(
data: &InterpData,
grammar_source: Option<&str>,
) -> io::Result<BTreeSet<usize>> {
let Some(grammar_source) = grammar_source else {
return Ok(BTreeSet::new());
};
Ok(assign_states_to_parser_action_slots(
data,
grammar_source,
parser_action_source_block_slots(grammar_source, &data.rule_names),
)?
.into_iter()
.filter_map(|(state, slot)| slot.body.trim().is_empty().then_some(state))
.collect())
}
/// Pairs supported rule-call arguments from grammar source with the ATN
/// rule-transition source states that carry those calls at runtime.
///
/// Runtime-test templates encode rule arguments in the original grammar text,
/// but the generated `.interp` data only preserves rule-transition structure.
/// Source order is stable for the covered fixtures, so matching grammar calls
/// to same-rule ATN transitions lets the generated parser expose local
/// predicate values without depending on ANTLR's Java code generator.
fn parser_rule_args(
data: &InterpData,
grammar_source: &str,
) -> io::Result<Vec<(usize, usize, RuleArgTemplate)>> {
let calls = literal_rule_arg_calls(data, grammar_source);
if calls.is_empty() {
return Ok(Vec::new());
}
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let mut rule_transitions = Vec::new();
for state in atn.states() {
for transition in state.transitions() {
if let ParserTransitionData::Rule { rule_index, .. } = transition.data() {
rule_transitions.push((state.state_number(), rule_index));
}
}
}
let mut used = vec![false; rule_transitions.len()];
let mut args = Vec::new();
for (rule_index, value) in calls {
if let Some((index, (source_state, _))) = rule_transitions
.iter()
.enumerate()
.find(|(index, (_, transition_rule))| !used[*index] && *transition_rule == rule_index)
{
used[index] = true;
args.push((*source_state, rule_index, value));
}
}
Ok(args)
}
/// Extracts calls like `a[2]` and `a[<VarRef("i")>]` while ignoring rule
/// declarations and target templates whose bracket contents are unsupported.
fn literal_rule_arg_calls(
data: &InterpData,
grammar_source: &str,
) -> Vec<(usize, RuleArgTemplate)> {
let rule_indices = data
.rule_names
.iter()
.enumerate()
.map(|(index, name)| (name.as_str(), index))
.collect::<BTreeMap<_, _>>();
let mut calls = Vec::new();
let mut cursor = templates::GrammarSourceCursor::new(grammar_source, 0);
while let Some((start, ch)) = cursor.next_significant() {
if ch == '{' {
cursor.seek(
matching_action_brace(grammar_source, start + 1)
.map_or(start + 1, |close| close + 1),
);
continue;
}
if ch != '_' && !ch.is_ascii_alphanumeric() {
continue;
}
let mut name_stop = start + ch.len_utf8();
while grammar_source
.as_bytes()
.get(name_stop)
.is_some_and(|byte| *byte == b'_' || byte.is_ascii_alphanumeric())
{
name_stop += 1;
}
cursor.seek(name_stop);
let Some(&rule_index) = rule_indices.get(&grammar_source[start..name_stop]) else {
continue;
};
let value_open = templates::skip_ascii_whitespace(grammar_source, name_stop);
if grammar_source.as_bytes().get(value_open) != Some(&b'[') {
continue;
}
let value_start = value_open + 1;
let Some(value_stop) = grammar_source[value_start..]
.find(']')
.map(|index| value_start + index)
else {
break;
};
let value = grammar_source[value_start..value_stop].trim();
let template = value.parse::<i64>().map_or_else(
|_| {
if matches!(value, "true" | "false") {
Some(RuleArgTemplate::Literal(i64::from(value == "true")))
} else if value == r#"<VarRef("i")>"# {
Some(RuleArgTemplate::InheritLocal)
} else {
None
}
},
|value| Some(RuleArgTemplate::Literal(value)),
);
if let Some(template) = template {
calls.push((start, rule_index, template));
}
cursor.seek(value_stop + 1);
}
calls.sort_by_key(|(start, _, _)| *start);
calls
.into_iter()
.map(|(_, rule_index, value)| (rule_index, value))
.collect()
}
/// Emits the generated lexer action dispatcher for grammar-specific custom
/// lexer actions discovered from the serialized ATN.
fn render_lexer_action_method(actions: &[((i32, i32), ActionTemplate)]) -> String {
if actions.is_empty() {
return String::new();
}
let mut comments = String::new();
for (_, template) in actions {
if let ActionTemplate::UnsupportedLexerAction { rule_name, body } = template {
writeln!(
comments,
" {}",
render_unsupported_lexer_action_comment(rule_name, body)
)
.expect("writing to a string cannot fail");
}
}
if !lexer_actions_need_dispatch(actions) {
return comments;
}
let mut arms = String::new();
for ((rule_index, action_index), template) in actions {
if !lexer_action_template_needs_dispatch(template) {
continue;
}
let statement = render_lexer_action_statement(template);
writeln!(
arms,
" ({rule_index}, {action_index}) => {{ {statement} true }}"
)
.expect("writing to a string cannot fail");
}
arms.push_str(" _ => false,\n");
format!(
"{comments} fn run_action(_base: &mut BaseLexer<I>, action: antlr4_runtime::LexerCustomAction) -> bool {{\n match (action.rule_index(), action.action_index()) {{\n{arms} }}\n }}\n"
)
}
fn lexer_actions_need_dispatch(actions: &[((i32, i32), ActionTemplate)]) -> bool {
actions
.iter()
.any(|(_, template)| lexer_action_template_needs_dispatch(template))
}
fn lexer_action_template_needs_dispatch(template: &ActionTemplate) -> bool {
match template {
ActionTemplate::Hook(_) | ActionTemplate::UnsupportedLexerAction { .. } => false,
ActionTemplate::LexerPopMode => !render_lexer_action_statement(template).is_empty(),
}
}
/// Renders one supported lexer target-template action as Rust code.
fn render_lexer_action_statement(template: &ActionTemplate) -> String {
match template {
ActionTemplate::LexerPopMode => "_base.pop_mode();".to_owned(),
ActionTemplate::Hook(_) => String::new(),
ActionTemplate::UnsupportedLexerAction { rule_name, body } => {
render_unsupported_lexer_action_comment(rule_name, body)
}
}
}
fn render_unsupported_lexer_action_comment(rule_name: &str, body: &str) -> String {
format!(
"/* TODO unsupported embedded lexer action in rule {}: {{{}}}; rewrite target-specific actions as portable lexer commands where possible */",
rust_block_comment_text(rule_name),
rust_block_comment_text(body)
)
}
/// Emits the generated lexer predicate dispatcher for grammar-specific
/// predicate coordinates discovered from the serialized ATN.
fn render_lexer_predicate_method(
predicates: &[((usize, usize), PredicateTemplate)],
sem_unknown: SemUnknownPolicy,
) -> String {
if predicates.is_empty() {
return String::new();
}
let mut arms = String::new();
for ((rule_index, pred_index), template) in predicates {
let statement = if matches!(template, PredicateTemplate::Hook) {
"None".to_owned()
} else {
format!("Some({})", render_lexer_predicate_expression(template))
};
writeln!(
arms,
" ({rule_index}, {pred_index}) => {{ {statement} }}"
)
.expect("writing to a string cannot fail");
}
// The catch-all arm is the unknown-lexer-predicate handler for any
// coordinate this grammar left untranslated. `--sem-unknown=assume-false`
// must flip it to `false` here too; otherwise a mixed lexer (one translated
// predicate plus an uncovered coordinate) keeps the uncovered guard viable
// even though the manifest promised assume-false.
let default_arm = if sem_unknown == SemUnknownPolicy::AssumeFalse {
" _ => Some(false),\n"
} else if sem_unknown == SemUnknownPolicy::Hook {
" _ => None,\n"
} else {
" _ => Some(true),\n"
};
arms.push_str(default_arm);
format!(
" fn run_predicate(_base: &BaseLexer<I>, predicate: antlr4_runtime::LexerPredicate) -> Option<bool> {{\n match (predicate.rule_index(), predicate.pred_index()) {{\n{arms} }}\n }}\n"
)
}
fn render_lexer_predicate_expression(template: &PredicateTemplate) -> String {
match template {
// A `<fail=...>` wrapper is transparent to evaluation; render its inner.
PredicateTemplate::WithFailMessage { inner, .. } => {
render_lexer_predicate_expression(inner)
}
PredicateTemplate::True => "true".to_owned(),
PredicateTemplate::False => "false".to_owned(),
PredicateTemplate::TextEquals(value) => format!(
"_base.token_text_until(predicate.position()) == \"{}\"",
rust_string(value)
),
PredicateTemplate::TokenStartColumnEquals(value) => {
format!("_base.token_start_column() == {value}")
}
PredicateTemplate::ColumnLessThan(value) => {
format!("_base.column_at(predicate.position()) < {value}")
}
PredicateTemplate::ColumnGreaterOrEqual(value) => {
format!("_base.column_at(predicate.position()) >= {value}")
}
PredicateTemplate::Hook
| PredicateTemplate::Invoke { .. }
| PredicateTemplate::FalseWithMessage { .. }
| PredicateTemplate::LocalIntEquals { .. }
| PredicateTemplate::LocalIntLessOrEqual { .. }
| PredicateTemplate::LookaheadTextEquals { .. }
| PredicateTemplate::LookaheadNotEquals { .. }
| PredicateTemplate::TokenPairAdjacent
| PredicateTemplate::ContextChildRuleTextNotEquals { .. } => {
unreachable!("lookahead parser predicates are not lexer predicates")
}
}
}
/// Reports whether a parser action source state carries an `assume-true` /
/// `assume-false` override. Such a coordinate is a documented silent no-op:
/// it must NOT fall through to the `parser_action_hook` catch-all (which fails
/// loud under the Error policy or runs a user side effect). It gets an explicit
/// empty arm instead. A `hook` (or `error`) override is excluded here so it
/// still routes to the hook.
fn parser_action_assume_overridden(
patterns: &SemPatternFile,
data: &InterpData,
action_state_rules: &BTreeMap<usize, usize>,
state: usize,
) -> bool {
let rule_name = action_state_rules
.get(&state)
.and_then(|rule| data.rule_names.get(*rule).map(String::as_str));
patterns
.coordinate_override(SemanticsKind::ParserAction, rule_name, None, Some(state))
.is_some_and(|override_| {
matches!(
override_.dispose,
CoordinateDispose::AssumeTrue | CoordinateDispose::AssumeFalse
)
})
}
fn render_parser_action_method(has_action_states: bool, noop_states: &BTreeSet<usize>) -> String {
if !has_action_states {
return " fn run_action(&mut self, _action: antlr4_runtime::ParserAction, _tree: antlr4_runtime::ParseTree) {}\n"
.to_owned();
}
let mut arms = String::new();
for state in noop_states {
writeln!(arms, " {state} => {{}}").expect("writing to a string cannot fail");
}
arms.push_str(" _ => { let _ = self.base.parser_action_hook(action, tree); }\n");
format!(
" fn run_action(&mut self, action: antlr4_runtime::ParserAction, tree: antlr4_runtime::ParseTree) {{\n match action.source_state() {{\n{arms} }}\n }}\n"
)
}
fn likely_parser_entry_rule_indices(data: &InterpData) -> io::Result<Vec<usize>> {
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&data.atn))
.deserialize_parser()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
Ok(likely_parser_entry_rule_indices_from_atn(
&atn,
data.rule_names.len(),
))
}
fn likely_parser_entry_rule_indices_from_atn(atn: &ParserAtn, rule_count: usize) -> Vec<usize> {
let mut called_by_other_rule = vec![false; rule_count];
for state in atn.states() {
for transition in state.transitions() {
let ParserTransitionData::Rule { rule_index, .. } = transition.data() else {
continue;
};
if rule_index >= rule_count || state.rule_index() == Some(rule_index) {
continue;
}
called_by_other_rule[rule_index] = true;
}
}
called_by_other_rule
.iter()
.enumerate()
.filter_map(|(index, called)| (!called).then_some(index))
.collect()
}
/// Renders the generated parser type rustdoc that surfaces callable rule methods.
fn render_parser_rustdoc(
public_rule_method_names: &[String],
entry_rule_indices: &[usize],
) -> String {
let all_method_capacity = public_rule_method_names
.iter()
.map(|method| method.len() + "/// - `()`\n".len())
.sum::<usize>();
let entry_method_capacity = entry_rule_indices
.iter()
.filter_map(|index| public_rule_method_names.get(*index))
.map(|method| method.len() + "/// - `()`\n".len())
.sum::<usize>();
let mut out = String::with_capacity(384 + all_method_capacity + entry_method_capacity);
writeln!(
out,
"/// Generated parser. Each grammar rule is exposed as a public method."
)
.expect("writing to a string cannot fail");
writeln!(out, "///").expect("writing to a string cannot fail");
writeln!(
out,
"/// Pick an entry-rule method that matches the grammar's intended"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"/// top-level construct for the input being parsed. The generator can"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"/// identify rules that are not called by another rule, but it cannot"
)
.expect("writing to a string cannot fail");
writeln!(
out,
"/// infer the semantic choice between multiple candidates."
)
.expect("writing to a string cannot fail");
if !entry_rule_indices.is_empty() {
writeln!(out, "///").expect("writing to a string cannot fail");
writeln!(
out,
"/// Likely parser entry-rule methods (not called by other rules):"
)
.expect("writing to a string cannot fail");
for index in entry_rule_indices {
let Some(method_name) = public_rule_method_names.get(*index) else {
continue;
};
writeln!(out, "/// - `{method_name}()`").expect("writing to a string cannot fail");
}
}
if !public_rule_method_names.is_empty() {
writeln!(out, "///").expect("writing to a string cannot fail");
writeln!(out, "/// All parser rule methods:").expect("writing to a string cannot fail");
for method_name in public_rule_method_names {
writeln!(out, "/// - `{method_name}()`").expect("writing to a string cannot fail");
}
}
out
}
/// Renders static grammar metadata shared by generated lexers and parsers.
fn render_metadata(grammar_name: &str, data: &InterpData) -> String {
render_metadata_with_atn(grammar_name, data, &data.atn)
}
/// Parser modules carry the versioned packed ATN separately, so retaining the
/// legacy serialized integer stream in metadata would duplicate the artifact.
fn render_parser_metadata(grammar_name: &str, data: &InterpData) -> String {
render_metadata_with_atn(grammar_name, data, &[])
}
fn render_metadata_with_atn(
grammar_name: &str,
data: &InterpData,
serialized_atn: &[i32],
) -> String {
format!(
"pub static METADATA: GrammarMetadata = GrammarMetadata::new(\n \"{}\",\n &{},\n &{},\n &{},\n &{},\n &{},\n &{},\n &{},\n);\n\npub fn metadata() -> &'static GrammarMetadata {{\n &METADATA\n}}\n\npub fn rule_names() -> &'static [&'static str] {{\n METADATA.rule_names()\n}}\n",
rust_string(grammar_name),
render_str_slice(&data.rule_names),
render_option_str_slice(&data.literal_names),
render_option_str_slice(&data.symbolic_names),
render_empty_option_str_slice(max_len(&data.literal_names, &data.symbolic_names)),
render_str_slice(&data.channel_names),
render_str_slice(&data.mode_names),
render_i32_slice(serialized_atn)
)
}
/// Renders token constants from symbolic token names while avoiding duplicate
/// Rust identifiers after sanitization.
fn render_token_constants(data: &InterpData) -> String {
let mut out = String::from("pub const EOF: i32 = antlr4_runtime::TOKEN_EOF;\n");
let mut seen = BTreeSet::new();
for (index, name) in data.symbolic_names.iter().enumerate() {
let Some(name) = name else { continue };
let ident = rust_const_name(name);
if ident == "EOF" || !seen.insert(ident.clone()) {
continue;
}
writeln!(out, "pub const {ident}: i32 = {index};")
.expect("writing to a string cannot fail");
}
out
}
/// Renders rule-index constants from grammar rule names.
fn render_rule_constants(data: &InterpData) -> String {
let mut out = String::new();
for (index, name) in data.rule_names.iter().enumerate() {
writeln!(
out,
"pub const RULE_{}: usize = {index};",
rust_const_name(name)
)
.expect("writing to a string cannot fail");
}
out
}
/// Renders an `&[Option<&str>]` expression for literal or symbolic names.
fn render_option_str_slice(values: &[Option<String>]) -> String {
let items = values
.iter()
.map(|value| {
value.as_ref().map_or_else(
|| "None".to_owned(),
|value| format!("Some(\"{}\")", rust_string(value)),
)
})
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders an empty optional string table with a fixed length.
fn render_empty_option_str_slice(len: usize) -> String {
let items = (0..len).map(|_| "None").collect::<Vec<_>>().join(", ");
format!("[{items}]")
}
/// Renders an `&[&str]` expression for rule/channel/mode names.
fn render_str_slice(values: &[String]) -> String {
let items = values
.iter()
.map(|value| format!("\"{}\"", rust_string(value)))
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders a line-wrapped `&[i32]` expression for serialized ATN data.
fn render_i32_slice(values: &[i32]) -> String {
let items = values
.iter()
.map(i32::to_string)
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders a versioned packed parser ATN word stream.
fn render_u32_slice(values: &[u32]) -> String {
let items = values
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders an inline `[(i32, i32); N]` expression for generated token-set
/// matches.
fn render_i32_ranges(values: &[(i32, i32)]) -> String {
let items = values
.iter()
.map(|(start, stop)| format!("({start}, {stop})"))
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
fn render_i32_match_patterns(values: &[(i32, i32)]) -> String {
values
.iter()
.map(|(start, stop)| {
if start == stop {
start.to_string()
} else {
format!("{start}..={stop}")
}
})
.collect::<Vec<_>>()
.join(" | ")
}
/// Renders parser predicate metadata shared by generated predicate checks.
#[allow(dead_code)]
fn render_parser_predicate_constant(
predicates: &[((usize, usize), PredicateTemplate)],
data: &InterpData,
) -> io::Result<String> {
let predicates = render_parser_predicate_array(predicates, data)?;
Ok(format!(
"#[allow(dead_code)]\nconst PARSER_PREDICATES: &[(usize, usize, antlr4_runtime::ParserPredicate)] = &{predicates};\n"
))
}
fn render_parser_semantics_function(
predicates: &[((usize, usize), PredicateTemplate)],
data: &InterpData,
) -> io::Result<String> {
let predicate_builders = render_parser_semir_predicate_builders(predicates, data)?;
Ok(format!(
r#"fn parser_semantics() -> &'static antlr4_runtime::ParserSemantics {{
static SEMANTICS_CELL: OnceLock<antlr4_runtime::ParserSemantics> = OnceLock::new();
SEMANTICS_CELL.get_or_init(|| {{
let mut ir = antlr4_runtime::semir::SemIr::new();
let mut predicates = Vec::new();
{predicate_builders}
let actions = Vec::new();
antlr4_runtime::ParserSemantics {{ ir, predicates, actions }}
}})
}}
"#
))
}
fn lexer_typed_hook_mappings(
data: &InterpData,
grammar_source: Option<&str>,
patterns: &SemPatternFile,
actions: &[((i32, i32), ActionTemplate)],
) -> io::Result<Vec<LexerTypedHookMapping>> {
let Some(grammar_source) = grammar_source else {
return Ok(Vec::new());
};
let mut mappings = actions
.iter()
.filter_map(|((rule_index, action_index), template)| {
let ActionTemplate::Hook(call) = template else {
return None;
};
Some(LexerTypedHookMapping {
rule_index: usize::try_from(*rule_index).ok()?,
coordinate_index: usize::try_from(*action_index).ok()?,
kind: LexerTypedHookKind::Action,
method_name: rust_function_name(&call.name),
call: call.clone(),
})
})
.collect::<Vec<_>>();
let coordinates = lexer_predicate_transitions(data)?;
let mut predicate_index = 0;
let mut offset = 0;
while let Some(block) = next_predicate_action_block(grammar_source, offset) {
offset = block.after_brace;
if !predicate_block_included(grammar_source, block.open_brace, &data.rule_names) {
continue;
}
let Some((rule_index, pred_index)) = coordinates.get(predicate_index).copied() else {
predicate_index += 1;
continue;
};
if let Some(call) = patterns.hook_helper_call(SemanticsKind::LexerPredicate, block.body)? {
mappings.push(LexerTypedHookMapping {
rule_index,
coordinate_index: pred_index,
kind: LexerTypedHookKind::Predicate,
method_name: rust_function_name(&call.name),
call,
});
}
predicate_index += 1;
}
let predicate_names = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Predicate)
.map(|mapping| mapping.method_name.clone())
.collect::<BTreeSet<_>>();
let action_names = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Action)
.map(|mapping| mapping.method_name.clone())
.collect::<BTreeSet<_>>();
for mapping in &mut mappings {
if predicate_names.contains(&mapping.method_name)
&& action_names.contains(&mapping.method_name)
{
mapping.method_name.push_str(match mapping.kind {
LexerTypedHookKind::Predicate => "_pred",
LexerTypedHookKind::Action => "_action",
});
}
}
validate_lexer_typed_hook_signatures(&mappings)?;
mappings.sort_by_key(|mapping| {
(
mapping.rule_index,
mapping.coordinate_index,
matches!(mapping.kind, LexerTypedHookKind::Action),
)
});
mappings.dedup();
Ok(mappings)
}
fn validate_lexer_typed_hook_signatures(mappings: &[LexerTypedHookMapping]) -> io::Result<()> {
let mut signatures = BTreeMap::<(&str, LexerTypedHookKind), Vec<SemanticLiteralKind>>::new();
for mapping in mappings {
let signature = mapping
.call
.arguments
.iter()
.map(semantic_literal_kind)
.collect::<Vec<_>>();
match signatures.entry((&mapping.method_name, mapping.kind)) {
Entry::Occupied(entry) if entry.get() != &signature => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"typed lexer semantic helper {} has conflicting literal signatures {:?} and {signature:?}",
mapping.call.name,
entry.get()
),
));
}
Entry::Occupied(_) => {}
Entry::Vacant(entry) => {
entry.insert(signature);
}
}
}
Ok(())
}
fn render_lexer_typed_hook_adapter(type_name: &str, mappings: &[LexerTypedHookMapping]) -> String {
if mappings.is_empty() {
return String::new();
}
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
let mut methods = BTreeMap::<(String, bool), Vec<SemanticLiteral>>::new();
for mapping in mappings {
methods
.entry((
mapping.method_name.clone(),
mapping.kind == LexerTypedHookKind::Predicate,
))
.or_insert_with(|| mapping.call.arguments.clone());
}
let method_decls = methods
.iter()
.map(|((method, predicate), arguments)| {
let arguments = render_semantic_method_arguments(arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let result = if *predicate { " -> bool" } else { "" };
format!(
" fn {method}<I>(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>{separator}{arguments}){result}\n where\n I: antlr4_runtime::CharStream;"
)
})
.collect::<Vec<_>>()
.join("\n\n");
let predicate_arms = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Predicate)
.map(|mapping| {
let rule = mapping.rule_index;
let index = mapping.coordinate_index;
let arguments = render_semantic_call_arguments(&mapping.call.arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let method = &mapping.method_name;
let call = format!("self.0.{method}(ctx{separator}{arguments})");
let call = if mapping.call.negated {
format!("!{call}")
} else {
call
};
format!(" ({rule}, {index}) => Some({call}),")
})
.collect::<Vec<_>>()
.join("\n");
let action_arms = mappings
.iter()
.filter(|mapping| mapping.kind == LexerTypedHookKind::Action)
.map(|mapping| {
let rule = mapping.rule_index;
let index = mapping.coordinate_index;
let arguments = render_semantic_call_arguments(&mapping.call.arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let method = &mapping.method_name;
format!(
" ({rule}, {index}) => {{ self.0.{method}(ctx{separator}{arguments}); true }}"
)
})
.collect::<Vec<_>>()
.join("\n");
format!(
r#"pub trait {trait_name}: Sized {{
{method_decls}
fn token_emitted(&mut self, _token: antlr4_runtime::TokenView<'_>) {{}}
}}
#[derive(Clone, Debug, Default)]
pub struct {adapter_name}<T>(pub T);
impl<T> {adapter_name}<T> {{
pub const fn new(inner: T) -> Self {{ Self(inner) }}
}}
impl<T> antlr4_runtime::SemanticHooks for {adapter_name}<T>
where
T: {trait_name},
{{
fn lexer_sempred<I>(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>, rule_index: usize, pred_index: usize) -> Option<bool>
where
I: antlr4_runtime::CharStream,
{{
match (rule_index, pred_index) {{
{predicate_arms}
_ => None,
}}
}}
fn lexer_action<I>(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>, action: antlr4_runtime::LexerCustomAction) -> bool
where
I: antlr4_runtime::CharStream,
{{
let Ok(rule_index) = usize::try_from(action.rule_index()) else {{ return false; }};
let Ok(action_index) = usize::try_from(action.action_index()) else {{ return false; }};
match (rule_index, action_index) {{
{action_arms}
_ => false,
}}
}}
fn lexer_token_emitted(&mut self, token: antlr4_runtime::TokenView<'_>) {{
self.0.token_emitted(token);
}}
}}
"#
)
}
fn parser_typed_hook_mappings(
data: &InterpData,
grammar_source: Option<&str>,
patterns: &SemPatternFile,
) -> io::Result<Vec<TypedHookMapping>> {
let Some(grammar_source) = grammar_source else {
return Ok(Vec::new());
};
let coordinates = parser_predicate_transitions(data)?;
let mut mappings = Vec::new();
let mut offset = 0;
let mut predicate_index = 0;
while let Some(block) = next_predicate_action_block(grammar_source, offset) {
offset = block.after_brace;
// Skip predicate blocks belonging to a different rule set (e.g. a
// lexer-rule predicate in a combined grammar), so the typed-hook mapping
// does not consume a parser coordinate for a non-parser block and wire
// the adapter to the wrong method. Matches `parser_predicate_templates`.
if !predicate_block_included(grammar_source, block.open_brace, &data.rule_names) {
continue;
}
let Some((rule_index, pred_index)) = coordinates.get(predicate_index).copied() else {
predicate_index += 1;
continue;
};
let helper_call = parse_semantic_helper_call(block.body, SemanticsKind::ParserPredicate);
let forced_hook = patterns
.coordinate_predicate_template(
SemanticsKind::ParserPredicate,
data.rule_names.get(rule_index).map(String::as_str),
Some(pred_index),
)
.is_some_and(|template| matches!(template, Some(PredicateTemplate::Hook)));
let parsed = parse_predicate_template_with_patterns(block.body, patterns)?;
if let Some(call) = helper_call
&& (forced_hook || parsed.is_none() || matches!(parsed, Some(PredicateTemplate::Hook)))
{
let method_name = typed_hook_predicate_method_name(&call.name);
mappings.push(TypedHookMapping {
rule_index,
pred_index,
method_name,
call,
});
}
predicate_index += 1;
}
mappings.sort_by_key(|mapping| (mapping.rule_index, mapping.pred_index));
mappings.dedup();
validate_typed_hook_signatures(&mappings)?;
Ok(mappings)
}
/// Reserved name of the fixed action-hook method emitted on the typed-hook
/// trait. A predicate-helper method must not normalize to this, or the trait
/// would declare two `custom_action` methods (Rust has no arity overloading).
const TYPED_HOOK_ACTION_METHOD: &str = "custom_action";
/// The typed-hook trait method name for a bare predicate helper, disambiguated
/// so it never collides with the fixed [`TYPED_HOOK_ACTION_METHOD`]. A grammar
/// helper literally named `customAction()` / `custom_action()` normalizes to
/// `custom_action`; suffix it with `_pred` so the generated trait compiles.
fn typed_hook_predicate_method_name(helper: &str) -> String {
let name = rust_function_name(helper);
if name == TYPED_HOOK_ACTION_METHOD {
format!("{name}_pred")
} else {
name
}
}
const fn semantic_literal_kind(literal: &SemanticLiteral) -> SemanticLiteralKind {
match literal {
SemanticLiteral::String(_) => SemanticLiteralKind::String,
SemanticLiteral::Bool(_) => SemanticLiteralKind::Bool,
SemanticLiteral::Integer(_) => SemanticLiteralKind::Integer,
}
}
fn validate_typed_hook_signatures(mappings: &[TypedHookMapping]) -> io::Result<()> {
let mut signatures = BTreeMap::<&str, Vec<SemanticLiteralKind>>::new();
for mapping in mappings {
let signature = mapping
.call
.arguments
.iter()
.map(semantic_literal_kind)
.collect::<Vec<_>>();
match signatures.entry(&mapping.method_name) {
Entry::Occupied(entry) if entry.get() != &signature => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"typed semantic helper {} has conflicting literal signatures {:?} and {signature:?}",
mapping.call.name,
entry.get()
),
));
}
Entry::Occupied(_) => {}
Entry::Vacant(entry) => {
entry.insert(signature);
}
}
}
Ok(())
}
fn render_semantic_method_arguments(arguments: &[SemanticLiteral]) -> String {
arguments
.iter()
.enumerate()
.map(|(index, literal)| {
let ty = match literal {
SemanticLiteral::String(_) => "&str",
SemanticLiteral::Bool(_) => "bool",
SemanticLiteral::Integer(_) => "i64",
};
format!("arg{index}: {ty}")
})
.collect::<Vec<_>>()
.join(", ")
}
fn render_semantic_call_arguments(arguments: &[SemanticLiteral]) -> String {
arguments
.iter()
.map(|literal| match literal {
SemanticLiteral::String(value) => format!("\"{}\"", rust_string(value)),
SemanticLiteral::Bool(value) => value.to_string(),
SemanticLiteral::Integer(value) => value.to_string(),
})
.collect::<Vec<_>>()
.join(", ")
}
fn render_typed_hook_adapter(type_name: &str, mappings: &[TypedHookMapping]) -> String {
if mappings.is_empty() {
return String::new();
}
let trait_name = format!("{type_name}Hooks");
let adapter_name = format!("{type_name}TypedHooks");
let mut methods = BTreeMap::new();
for mapping in mappings {
methods
.entry(mapping.method_name.clone())
.or_insert_with(|| mapping.call.arguments.clone());
}
let method_decls = methods
.iter()
.map(|(method, arguments)| {
let arguments = render_semantic_method_arguments(arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
format!(
" fn {method}<L>(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>{separator}{arguments}) -> bool\n where\n L: TokenSource;"
)
})
.collect::<Vec<_>>()
.join("\n\n");
let arms = mappings
.iter()
.map(|mapping| {
let rule_index = mapping.rule_index;
let pred_index = mapping.pred_index;
let method = &mapping.method_name;
let arguments = render_semantic_call_arguments(&mapping.call.arguments);
let separator = if arguments.is_empty() { "" } else { ", " };
let call = format!("self.0.{method}(ctx{separator}{arguments})");
let call = if mapping.call.negated {
format!("!{call}")
} else {
call
};
format!(" ({rule_index}, {pred_index}) => Some({call}),")
})
.collect::<Vec<_>>()
.join("\n");
format!(
r#"pub trait {trait_name}: Sized {{
{method_decls}
/// Handles a committed parser action routed to the typed hook. Return
/// `true` when the action is handled so it satisfies a `hook`/`error`
/// unknown-semantic policy; the default no-op returns `false` (unhandled),
/// which fails loud under those policies.
fn custom_action<L>(&mut self, _ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, _action: antlr4_runtime::ParserAction) -> bool
where
L: TokenSource,
{{
false
}}
}}
#[derive(Clone, Copy, Debug, Default)]
pub struct {adapter_name}<T>(pub T);
impl<T> {adapter_name}<T> {{
pub const fn new(inner: T) -> Self {{ Self(inner) }}
}}
impl<T> antlr4_runtime::SemanticHooks for {adapter_name}<T>
where
T: {trait_name},
{{
fn sempred<L>(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, rule_index: usize, pred_index: usize) -> Option<bool>
where
L: TokenSource,
{{
match (rule_index, pred_index) {{
{arms}
_ => None,
}}
}}
fn action<L>(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, action: antlr4_runtime::ParserAction) -> bool
where
L: TokenSource,
{{
self.0.custom_action(ctx, action)
}}
}}
"#
)
}
fn render_parser_semir_predicate_builders(
predicates: &[((usize, usize), PredicateTemplate)],
data: &InterpData,
) -> io::Result<String> {
let mut out = String::new();
for ((rule_index, pred_index), predicate) in predicates {
let expr = render_parser_semir_predicate_expr(predicate, data)?;
// Carry the `<fail=...>` message for ANY predicate that supplies one, not
// only a constant-false one — a hook/lookahead/member predicate that
// returns false at runtime should surface the grammar's fail text too.
let failure_message = predicate_template_fail_message(predicate).map_or_else(
|| "None".to_owned(),
|message| format!("Some(\"{}\")", rust_string(message)),
);
writeln!(
out,
" let __expr = {expr};\n predicates.push(antlr4_runtime::ParserSemanticPredicate {{ rule_index: {rule_index}, pred_index: {pred_index}, expr: __expr, failure_message: {failure_message} }});"
)
.expect("writing to a string cannot fail");
}
Ok(out)
}
#[allow(clippy::too_many_lines)]
fn render_parser_semir_predicate_expr(
predicate: &PredicateTemplate,
data: &InterpData,
) -> io::Result<String> {
match predicate {
// A `<fail=...>` wrapper is transparent to evaluation; lower its inner.
PredicateTemplate::WithFailMessage { inner, .. } => {
render_parser_semir_predicate_expr(inner, data)
}
PredicateTemplate::Hook => Ok(
"ir.expr(antlr4_runtime::semir::PExpr::Hook(antlr4_runtime::semir::HookId::new(0)))"
.to_owned(),
),
PredicateTemplate::True => {
Ok("ir.expr(antlr4_runtime::semir::PExpr::Bool(true))".to_owned())
}
PredicateTemplate::False | PredicateTemplate::FalseWithMessage { .. } => {
Ok("ir.expr(antlr4_runtime::semir::PExpr::Bool(false))".to_owned())
}
PredicateTemplate::Invoke { value } => Ok(format!(
"ir.expr(antlr4_runtime::semir::PExpr::EvalTrace({value}))"
)),
PredicateTemplate::LocalIntEquals { value } => Ok(render_local_arg_semir_cmp("Eq", *value)),
PredicateTemplate::LocalIntLessOrEqual { value } => {
Ok(render_local_arg_semir_cmp("Le", *value))
}
PredicateTemplate::LookaheadTextEquals { offset, text } => Ok(format!(
"{{ let __actual = ir.expr(antlr4_runtime::semir::PExpr::TokenText({offset})); let __text = ir.intern(\"{}\"); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Str(__text)); ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::Eq, __actual, __expected)) }}",
rust_string(text)
)),
PredicateTemplate::LookaheadNotEquals { offset, token_name } => {
let token_type = token_type_for_name(data, token_name).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate token {token_name}"),
)
})?;
Ok(format!(
"{{ let __actual = ir.expr(antlr4_runtime::semir::PExpr::La({offset})); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Int({token_type})); ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::Ne, __actual, __expected)) }}"
))
}
PredicateTemplate::TokenPairAdjacent => {
Ok("ir.expr(antlr4_runtime::semir::PExpr::TokenIndexAdjacent)".to_owned())
}
PredicateTemplate::ContextChildRuleTextNotEquals { rule_name, text } => {
let rule_index = data
.rule_names
.iter()
.position(|name| name == rule_name)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate rule {rule_name}"),
)
})?;
Ok(format!(
"{{ let __actual = ir.expr(antlr4_runtime::semir::PExpr::CtxRuleText({rule_index})); let __text = ir.intern(\"{}\"); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Str(__text)); ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::Ne, __actual, __expected)) }}",
rust_string(text)
))
}
PredicateTemplate::TextEquals(_)
| PredicateTemplate::TokenStartColumnEquals(_)
| PredicateTemplate::ColumnLessThan(_)
| PredicateTemplate::ColumnGreaterOrEqual(_) => Err(io::Error::new(
io::ErrorKind::InvalidData,
"lexer-only predicate cannot be lowered for parser SemIR",
)),
}
}
fn render_local_arg_semir_cmp(op: &str, value: i64) -> String {
format!(
"{{ let __local = ir.expr(antlr4_runtime::semir::PExpr::LocalArg); let __absent = ir.expr(antlr4_runtime::semir::PExpr::IsNull(__local)); let __expected = ir.expr(antlr4_runtime::semir::PExpr::Int({value})); let __comparison = ir.expr(antlr4_runtime::semir::PExpr::Cmp(antlr4_runtime::semir::CmpOp::{op}, __local, __expected)); ir.expr(antlr4_runtime::semir::PExpr::Or([__absent, __comparison].into())) }}"
)
}
/// Renders parser predicate metadata as an inline slice consumed by the runtime
/// parser interpreter.
#[allow(dead_code)]
fn render_parser_predicate_array(
predicates: &[((usize, usize), PredicateTemplate)],
data: &InterpData,
) -> io::Result<String> {
let mut items = Vec::new();
for ((rule_index, pred_index), predicate) in predicates {
// The deprecated `ParserPredicate` table (SemIR is the active path). A
// `<fail=...>` wrapper on a non-constant-false predicate has no encoding
// in this legacy enum, so render the transparent inner template; the
// SemIR predicate builder carries the message on the active path.
let predicate = predicate_effective_template(predicate);
let expression = match predicate {
PredicateTemplate::True => "antlr4_runtime::ParserPredicate::True".to_owned(),
PredicateTemplate::Hook => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"hook predicates lower only through parser SemIR",
));
}
PredicateTemplate::False => "antlr4_runtime::ParserPredicate::False".to_owned(),
PredicateTemplate::FalseWithMessage { message } => {
format!(
"antlr4_runtime::ParserPredicate::FalseWithMessage {{ message: \"{}\" }}",
rust_string(message)
)
}
PredicateTemplate::Invoke { value } => {
format!("antlr4_runtime::ParserPredicate::Invoke {{ value: {value} }}")
}
PredicateTemplate::LocalIntEquals { value } => {
format!("antlr4_runtime::ParserPredicate::LocalIntEquals {{ value: {value} }}")
}
PredicateTemplate::LocalIntLessOrEqual { value } => {
format!("antlr4_runtime::ParserPredicate::LocalIntLessOrEqual {{ value: {value} }}")
}
PredicateTemplate::TextEquals(_) => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"TextEquals is only supported for lexer predicates",
));
}
PredicateTemplate::TokenStartColumnEquals(_)
| PredicateTemplate::ColumnLessThan(_)
| PredicateTemplate::ColumnGreaterOrEqual(_) => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"column predicates are only supported for lexer predicates",
));
}
PredicateTemplate::LookaheadTextEquals { offset, text } => {
format!(
"antlr4_runtime::ParserPredicate::LookaheadTextEquals {{ offset: {offset}, text: \"{}\" }}",
rust_string(text)
)
}
PredicateTemplate::LookaheadNotEquals { offset, token_name } => {
let token_type = token_type_for_name(data, token_name).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate token {token_name}"),
)
})?;
format!(
"antlr4_runtime::ParserPredicate::LookaheadNotEquals {{ offset: {offset}, token_type: {token_type} }}"
)
}
PredicateTemplate::TokenPairAdjacent => {
"antlr4_runtime::ParserPredicate::TokenPairAdjacent".to_owned()
}
PredicateTemplate::ContextChildRuleTextNotEquals { rule_name, text } => {
let rule_index = data
.rule_names
.iter()
.position(|name| name == rule_name)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown predicate rule {rule_name}"),
)
})?;
format!(
"antlr4_runtime::ParserPredicate::ContextChildRuleTextNotEquals {{ rule_index: {rule_index}, text: \"{}\" }}",
rust_string(text)
)
}
// `predicate_effective_template` above already unwrapped any wrapper;
// the constructor never nests, so this is unreachable.
PredicateTemplate::WithFailMessage { .. } => {
unreachable!("predicate_effective_template unwraps the fail-message wrapper")
}
};
items.push(format!("({rule_index}, {pred_index}, {expression})"));
}
Ok(format!("[{}]", items.join(", ")))
}
/// Renders parser rule-argument metadata for generated calls into the runtime.
fn render_parser_rule_arg_array(args: &[(usize, usize, RuleArgTemplate)]) -> String {
let items = args
.iter()
.map(|(source_state, rule_index, value)| {
let (value, inherit_local) = match value {
RuleArgTemplate::Literal(value) => (*value, false),
RuleArgTemplate::InheritLocal => (0, true),
};
format!(
"antlr4_runtime::ParserRuleArg {{ source_state: {source_state}, rule_index: {rule_index}, value: {value}, inherit_local: {inherit_local} }}"
)
})
.collect::<Vec<_>>()
.join(", ");
format!("[{items}]")
}
/// Renders the generated parser base construction.
///
/// When a non-default unknown-predicate policy is configured, the constructor
/// installs it on the `BaseParser` so the generated recursive-descent path
/// (which evaluates predicates without going through `ParserRuntimeOptions`)
/// honors `--sem-unknown` too, rather than leaving the field at `AssumeTrue`.
fn render_parser_base_initialization(unknown_policy_literal: Option<&str>) -> String {
let needs_mut = unknown_policy_literal.is_some();
let mut out = if needs_mut {
" let mut base = BaseParser::with_semantic_hooks(input, data, hooks);".to_owned()
} else {
" let base = BaseParser::with_semantic_hooks(input, data, hooks);".to_owned()
};
if let Some(policy) = unknown_policy_literal {
write!(
out,
"\n base.set_unknown_predicate_policy({policy});"
)
.expect("writing to a string cannot fail");
}
out
}
/// Renders the parser-module convenience that wires text input through the
/// caller-selected lexer, token stream, parser, and entry rule in one call.
fn render_parser_parse_convenience(type_name: &str) -> String {
let output_type_name = format!("{type_name}ParseOutput");
format!(
r#"/// Result from [`parse_with_parser`].
///
/// Keeps the generated parser available after the entry rule runs so callers
/// can inspect diagnostics or recover the parser-owned token stream.
#[derive(Debug)]
pub struct {output_type_name}<R, L>
where
L: TokenSource,
{{
pub result: R,
pub parser: {type_name}<L>,
}}
/// Parses UTF-8 text by constructing the lexer, token stream, parser, and
/// caller-selected entry rule in one call.
///
/// Pass the generated lexer constructor and a parser entry rule, for example
/// `parse(src, MyGrammarLexer::new, {type_name}::file)`.
///
/// The returned [`antlr4_runtime::ParsedFile`] owns the canonical token store,
/// flat CST storage, and entry-rule root.
/// Use [`parse_with_parser`] instead when the caller also needs parser
/// diagnostics after the entry rule runs.
pub fn parse<L: TokenSource>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<antlr4_runtime::NodeId, antlr4_runtime::AntlrError>,
) -> Result<antlr4_runtime::ParsedFile, antlr4_runtime::AntlrError>
{{
let {output_type_name} {{ result, parser }} = parse_with_parser(input, lexer, entry)?;
Ok(parser.into_parsed_file(result))
}}
/// Parses UTF-8 text like [`parse`] while returning the parser after the entry
/// rule has run.
///
/// This keeps the compact generated setup path available for callers that also
/// need `Parser::number_of_syntax_errors()` or `{type_name}::into_token_stream()`.
pub fn parse_with_parser<L: TokenSource, R>(
input: impl AsRef<str>,
lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
entry: impl FnOnce(&mut {type_name}<L>) -> Result<R, antlr4_runtime::AntlrError>,
) -> Result<{output_type_name}<R, L>, antlr4_runtime::AntlrError>
{{
let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref()));
let tokens = CommonTokenStream::new(lexer);
let mut parser = {type_name}::new(tokens);
let result = entry(&mut parser)?;
Ok({output_type_name} {{ result, parser }})
}}"#
)
}
fn token_type_for_name(data: &InterpData, token_name: &str) -> Option<usize> {
data.symbolic_names
.iter()
.position(|name| name.as_deref() == Some(token_name))
}
/// Converts an ANTLR token/rule name into an upper-snake Rust constant name.
fn rust_const_name(name: &str) -> String {
let words = split_identifier_words(name);
let ident = if words.is_empty() {
"TOKEN".to_owned()
} else {
ascii_uppercase(&words.join("_"))
};
sanitize_identifier(&ident)
}
/// Converts ASCII letters to upper case without using allocation-hiding string
/// case helpers disallowed by the strict Clippy policy.
fn ascii_uppercase(value: &str) -> String {
value.chars().map(|ch| ch.to_ascii_uppercase()).collect()
}
fn max_len(left: &[Option<String>], right: &[Option<String>]) -> usize {
left.len().max(right.len())
}
/// Derives a grammar name from an input file stem when the user does not pass
/// an explicit `--lexer-name` or `--parser-name`.
fn grammar_name_from_path(path: &Path) -> String {
path.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("Grammar")
.to_owned()
}
#[cfg(test)]
mod tests {
use super::*;
use antlr4_runtime::atn::parser_atn::{ParserAtnBuilder, ParserTransitionSpec};
#[test]
fn parses_interp_sections() {
let data = InterpData::parse(
r#"token literal names:
null
'x'
token symbolic names:
null
X
rule names:
file
channel names:
DEFAULT_TOKEN_CHANNEL
HIDDEN
mode names:
DEFAULT_MODE
atn:
[4, 1, 1, 0]
"#,
)
.expect("interp data should parse");
assert_eq!(data.literal_names[1], Some("'x'".to_owned()));
assert_eq!(data.symbolic_names[1], Some("X".to_owned()));
assert_eq!(data.rule_names, ["file"]);
assert_eq!(data.atn, [4, 1, 1, 0]);
}
#[test]
fn renders_module_level_metadata_helpers() {
let rendered = render_metadata("TParser", &minimal_parser_data());
assert!(
rendered.contains("pub fn metadata() -> &'static GrammarMetadata {\n &METADATA\n}")
);
assert!(rendered.contains(
"pub fn rule_names() -> &'static [&'static str] {\n METADATA.rule_names()\n}"
));
}
#[test]
fn converts_names_to_rust_identifiers() {
assert_eq!(module_name("ExprLexer"), "expr_lexer");
assert_eq!(rust_function_name("sourceFile"), "source_file");
assert_eq!(rust_const_name("LPAREN"), "LPAREN");
assert_eq!(rust_const_name("Q_COLONCOLON"), "Q_COLONCOLON");
assert_eq!(rust_const_name("LineStrExprStart"), "LINE_STR_EXPR_START");
assert_eq!(rust_const_name("UnicodeClassLL"), "UNICODE_CLASS_LL");
assert_eq!(rust_function_name("gen"), "r#gen");
assert_eq!(rust_function_name("try"), "r#try");
assert_eq!(rust_function_name("Self"), "r#self");
assert!(is_rust_keyword("Self"));
}
#[test]
fn renders_parser_rustdoc_with_entry_rule_methods() {
let data = InterpData {
rule_names: vec![
"sourceFile".to_owned(),
"declaration".to_owned(),
"script".to_owned(),
"try".to_owned(),
],
..InterpData::default()
};
let entry_rule_indices = vec![0, 2];
let rendered = render_parser_rustdoc(
&parser_public_rule_method_names(&data.rule_names),
&entry_rule_indices,
);
assert!(rendered.contains("Likely parser entry-rule methods"));
assert!(rendered.contains("/// - `source_file()`"));
assert!(rendered.contains("/// - `script()`"));
assert!(rendered.contains("All parser rule methods:"));
assert!(rendered.contains("/// - `declaration()`"));
assert!(rendered.contains("/// - `r#try()`"));
assert!(rendered.contains("cannot"));
assert!(rendered.contains("semantic choice"));
}
#[test]
fn infers_entry_rule_candidates_from_rule_call_graph() {
let atn = entry_candidate_atn();
assert_eq!(
likely_parser_entry_rule_indices_from_atn(&atn, 4),
vec![0, 2, 3]
);
}
#[test]
fn generated_parser_rustdoc_is_attached_to_parser_type() {
let rendered =
render_parser("DemoParser", &minimal_parser_data(), None).expect("parser renders");
assert!(rendered.contains(
"/// Generated parser. Each grammar rule is exposed as a public method.\n///\n/// Pick an entry-rule method"
));
assert!(rendered.contains(
"/// Likely parser entry-rule methods (not called by other rules):\n/// - `s()`"
));
assert!(rendered.contains(
"/// All parser rule methods:\n/// - `s()`\n#[derive(Debug)]\npub struct DemoParser<L, H = antlr4_runtime::NoSemanticHooks>"
));
}
#[test]
fn generated_parser_embeds_only_versioned_packed_atn_data() {
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(rendered.contains("static PARSER_ATN_DATA: &[u32]"));
assert!(rendered.contains("static ATN_CELL: OnceLock<ParserAtn>"));
assert!(rendered.contains("ParserAtn::from_static(PARSER_ATN_DATA)"));
assert!(rendered.contains("generated parser ATN is incompatible with this runtime"));
assert!(rendered.contains("pub fn parser_atn() -> &'static ParserAtn"));
assert!(rendered.contains("fn parser_atn() -> &'static ParserAtn"));
assert!(!rendered.contains("AtnDeserializer"));
assert!(!rendered.contains("SerializedAtn"));
}
#[test]
fn parser_rule_method_names_reserve_recognizer_reuse_accessors() {
let rule_names = vec![
"tokenStream".to_owned(),
"into_token_stream".to_owned(),
"token_stream_rule".to_owned(),
"reset".to_owned(),
"setTokenStream".to_owned(),
"clearDfa".to_owned(),
"addErrorListener".to_owned(),
"removeErrorListeners".to_owned(),
"regularRule".to_owned(),
];
assert_eq!(
parser_public_rule_method_names(&rule_names),
[
"token_stream_rule",
"into_token_stream_rule",
"token_stream_rule_2",
"reset_rule",
"set_token_stream_rule",
"clear_dfa_rule",
"add_error_listener_rule",
"remove_error_listeners_rule",
"regular_rule"
]
);
}
#[test]
fn generated_modules_start_with_file_level_header() {
let lexer = render_lexer(
"TLexer",
&predicate_lexer_data(),
None,
false,
SemUnknownPolicy::default(),
&SemPatternFile::default(),
false,
)
.expect("lexer module should render");
let parser =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(GENERATED_MODULE_HEADER.contains(concat!(
"@generated by ",
env!("CARGO_PKG_NAME"),
" v",
env!("CARGO_PKG_VERSION")
)));
assert!(GENERATED_MODULE_HEADER.contains(env!("CARGO_PKG_REPOSITORY")));
for rendered in [&lexer, &parser] {
assert!(rendered.starts_with(GENERATED_MODULE_HEADER));
assert!(rendered[GENERATED_MODULE_HEADER.len()..].starts_with("use antlr4_runtime::"));
assert!(!rendered.contains("#!["));
assert!(rendered.contains(GENERATED_MODULE_FOOTER));
assert!(
rendered.ends_with("pub use self::__antlr4_rust_generated::*;\n"),
"generated module should end with exactly one trailing newline and no blank line at EOF"
);
}
}
fn compile_test_parser_rule(
atn: &ParserAtn,
rule_index: usize,
inline_action_states: &BTreeSet<usize>,
) -> Option<GeneratedParserRule> {
let decision_by_state = decision_by_state(atn);
let action_states = BTreeSet::new();
let generated_action_states = BTreeSet::new();
let predicate_coordinates = BTreeSet::new();
let generated_predicate_coordinates = BTreeSet::new();
let context = GeneratedParserCompileContext {
atn,
decision_by_state: &decision_by_state,
rule_args: &[],
inline_action_states,
action_states: &action_states,
generated_action_states: &generated_action_states,
predicate_coordinates: &predicate_coordinates,
generated_predicate_coordinates: &generated_predicate_coordinates,
};
compile_generated_parser_rule(&context, rule_index)
}
fn finish_atn(builder: ParserAtnBuilder) -> ParserAtn {
builder.finish().expect("valid packed parser ATN")
}
fn transition_atn(
make_transition: impl FnOnce(&mut ParserAtnBuilder) -> ParserTransitionSpec,
) -> ParserAtn {
let mut builder = ParserAtnBuilder::new(10);
for _ in 0..10 {
builder
.add_state(AtnStateKind::Basic, Some(0))
.expect("state");
}
builder
.set_rule_to_start_state(vec![0, 0, 0])
.expect("rule start states");
builder
.set_rule_to_stop_state(vec![9, 9, 9])
.expect("rule stop states");
let transition = make_transition(&mut builder);
builder
.add_transition(0, transition)
.expect("test transition");
finish_atn(builder)
}
fn only_transition(atn: &ParserAtn) -> ParserTransition<'_> {
atn.state(0)
.expect("transition source")
.transitions()
.first()
.expect("test transition")
}
fn mt(token_type: i32, follow_state: usize) -> GeneratedParserStep {
GeneratedParserStep::MatchToken {
token_type,
follow_state,
}
}
fn ms(intervals: Vec<(i32, i32)>, follow_state: usize) -> GeneratedParserStep {
GeneratedParserStep::MatchSet {
token_set: None,
intervals,
follow_state,
}
}
fn mts(
token_set: usize,
intervals: Vec<(i32, i32)>,
follow_state: usize,
) -> GeneratedParserStep {
GeneratedParserStep::MatchSet {
token_set: Some(token_set),
intervals,
follow_state,
}
}
fn mnts(
token_set: usize,
intervals: Vec<(i32, i32)>,
follow_state: usize,
) -> GeneratedParserStep {
GeneratedParserStep::MatchNotSet {
token_set: Some(token_set),
intervals,
follow_state,
}
}
fn mns(intervals: Vec<(i32, i32)>, follow_state: usize) -> GeneratedParserStep {
GeneratedParserStep::MatchNotSet {
token_set: None,
intervals,
follow_state,
}
}
fn cr(rule_index: usize) -> GeneratedParserStep {
GeneratedParserStep::CallRule {
source_state: 100 + rule_index,
rule_index,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}
}
fn adaptive_loop(decision: usize) -> GeneratedParserStep {
GeneratedParserStep::StarLoop {
state: 1_000 + decision,
decision,
enter_alt: 1,
exit_alt: 2,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
plus_loop: false,
fast_path: None,
body: vec![mt(2, 0)],
}
}
fn expensive_ladder_rule(rule_index: usize, next: Option<usize>) -> GeneratedParserRule {
let mut steps = Vec::new();
if let Some(next) = next {
steps.push(cr(next));
}
steps.push(adaptive_loop(rule_index * 2));
steps.push(adaptive_loop(rule_index * 2 + 1));
if next.is_none() {
steps.push(mt(1, 0));
}
test_rule(rule_index, steps)
}
fn test_rule(rule_index: usize, steps: Vec<GeneratedParserStep>) -> GeneratedParserRule {
GeneratedParserRule {
rule_index,
entry_state: rule_index * 2,
left_recursive: false,
steps,
}
}
#[test]
fn compiles_linear_parser_rule_body() {
let atn = linear_rule_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("linear rule should compile");
assert_eq!(body.rule_index, 0);
assert_eq!(body.entry_state, 0);
assert_eq!(body.steps, [mt(1, 2), mt(TOKEN_EOF, 3)]);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
assert!(rendered.contains("match_token_recovering(1, 2, atn())"));
assert!(rendered.contains("generated_diagnostics_checkpoint()"));
assert!(rendered.contains("restore_generated_diagnostics(__generated_diagnostic_marker)"));
}
#[test]
fn compiles_block_decision_with_adaptive_prediction() {
let atn = block_decision_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("block decision rule should compile");
assert_eq!(
body.steps,
[GeneratedParserStep::Decision {
state: 1,
decision: 0,
track_alt_number: true,
allow_semantic_context: false,
force_context: false,
fast_path: Some(GeneratedDecisionFastPath {
arms: vec![
GeneratedDecisionFastArm {
alt: 1,
intervals: vec![(1, 1)],
},
GeneratedDecisionFastArm {
alt: 2,
intervals: vec![(2, 2)],
},
],
}),
alts: vec![vec![mt(1, 4)], vec![mt(2, 4)]],
}]
);
let rendered =
render_generated_rule_dispatch(&[Some(body.clone())], &[], &BTreeMap::new(), false);
assert!(rendered.contains("parse_generated_rule_0"));
assert!(rendered.contains("sync_decision(atn(), 1, !__ctx.has_matched_child(), false)"));
assert!(rendered.contains("ll1_decision_prediction(atn(), 1)"));
// Stage 1 is the SLL probe (no LL loop on the empty-context conflict);
// stage 2 re-runs with the real context only when full context is needed.
assert!(rendered.contains("adaptive_predict_stream_info_sll_probe(0, 0"));
assert!(rendered.contains("adaptive_predict_stream_info_with_context(0, 0"));
assert!(rendered.contains(
"intern_prediction_context(self.base.rule_context_version(), self.base.prediction_context_return_states(atn()))"
));
assert!(!rendered.contains("self.base.prediction_context(atn())"));
let rendered_with_alt_numbers =
render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), true);
assert!(rendered_with_alt_numbers.contains("__ctx.set_alt_number(1);"));
assert!(rendered_with_alt_numbers.contains("__ctx.set_alt_number(2);"));
}
#[test]
fn compiles_star_loop_with_adaptive_prediction() {
let atn = star_loop_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("star loop rule should compile");
assert_eq!(
body.steps,
[GeneratedParserStep::StarLoop {
state: 1,
decision: 0,
enter_alt: 1,
exit_alt: 2,
track_alt_number: true,
allow_semantic_context: false,
force_context: false,
plus_loop: false,
fast_path: None,
body: vec![mt(1, 4)],
}]
);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
assert!(rendered.contains("loop {"));
// A `*` loop starts NOT iterated: its first sync is at the loop entry
// (single-token deletion), so the iteration flag inits to `false`.
assert!(rendered.contains("let mut __loop_iter_1 = false;"));
assert!(
rendered.contains("sync_decision(atn(), 1, !__ctx.has_matched_child(), __loop_iter_1)")
);
assert!(rendered.contains("__loop_iter_1 = true;"));
assert!(rendered.contains("1 => {"));
assert!(rendered.contains("2 => {"));
assert!(rendered.contains("break;"));
assert!(rendered.contains("ll1_decision_prediction(atn(), 1)"));
assert!(rendered.contains("adaptive_predict_stream_info_sll_probe(0, 0"));
assert!(rendered.contains("adaptive_predict_stream_info_with_context(0, 0"));
}
#[test]
fn compiles_plus_loop_back_with_adaptive_prediction() {
let atn = plus_loop_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("plus loop rule should compile");
assert_eq!(
body.steps,
[
mt(1, 3),
GeneratedParserStep::StarLoop {
state: 4,
decision: 0,
enter_alt: 1,
exit_alt: 2,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
plus_loop: true,
fast_path: None,
body: vec![mt(1, 3)],
}
]
);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
// A `+` loop's mandatory first element is iteration 1, so the iteration
// flag inits to `true`: its first loop-back sync recovers with multi-token
// `consumeUntil`, matching ANTLR's PLUS_LOOP_BACK.
assert!(rendered.contains("let mut __loop_iter_4 = true;"));
assert!(
rendered.contains("sync_decision(atn(), 4, !__ctx.has_matched_child(), __loop_iter_4)")
);
}
#[test]
fn compiles_plus_block_body_decision_with_adaptive_prediction() {
let atn = plus_block_decision_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("plus block decision rule should compile");
let body_decision = GeneratedParserStep::Decision {
state: 1,
decision: 0,
track_alt_number: true,
allow_semantic_context: false,
force_context: false,
fast_path: Some(GeneratedDecisionFastPath {
arms: vec![
GeneratedDecisionFastArm {
alt: 1,
intervals: vec![(1, 1)],
},
GeneratedDecisionFastArm {
alt: 2,
intervals: vec![(2, 2)],
},
],
}),
alts: vec![vec![mt(1, 4)], vec![mt(2, 4)]],
};
assert_eq!(
body.steps,
[
body_decision.clone(),
GeneratedParserStep::StarLoop {
state: 5,
decision: 1,
enter_alt: 1,
exit_alt: 2,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
plus_loop: true,
fast_path: None,
body: vec![body_decision],
}
]
);
}
#[test]
fn compiles_left_recursive_parser_rule() {
let atn = left_recursive_rule_atn();
let body = compile_test_parser_rule(&atn, 0, &BTreeSet::new())
.expect("left-recursive rule should compile");
assert!(body.left_recursive);
assert_eq!(body.rule_index, 0);
assert_eq!(body.entry_state, 0);
assert_eq!(
body.steps,
[
mt(1, 2),
GeneratedParserStep::LeftRecursiveLoop {
state: 2,
decision: 0,
enter_alt: 1,
exit_alt: 2,
rule_index: 0,
entry_state: 0,
body: vec![GeneratedParserStep::Decision {
state: 3,
decision: 1,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: vec![vec![
GeneratedParserStep::Precedence(2),
mt(2, 10),
GeneratedParserStep::CallRule {
source_state: 10,
rule_index: 0,
precedence: GeneratedRuleCallPrecedence::Literal(3),
},
]],
}],
}
]
);
let rendered = render_generated_rule_dispatch(&[Some(body)], &[], &BTreeMap::new(), false);
assert!(rendered.contains("parse_generated_rule_0_precedence(precedence, allow_fallback)"));
assert!(
rendered.contains("push_new_recursion_context_with_previous(0isize, 0, &mut __ctx)")
);
assert!(rendered.contains("parse_rule_precedence_from_generated(0, 3)"));
assert!(rendered.contains("precpred(_ctx, 2)"));
assert!(
rendered.contains(
"let __prediction = match self.base.left_recursive_loop_enter_prediction(atn(), 2, __precedence)"
)
);
assert!(rendered.contains("Some(true) => antlr4_runtime::ParserAtnPrediction"));
assert!(
rendered
.contains("adaptive_predict_stream_info_with_context(0, __prediction_precedence")
);
assert!(rendered.contains(
"Err(antlr4_runtime::ParserAtnSimulatorError::NoViableAlt { .. }) => antlr4_runtime::ParserAtnPrediction { alt: 2, requires_full_context: true, has_semantic_context: false, diagnostic: None }"
));
}
#[test]
fn drops_generated_rules_that_call_disabled_rules() {
let mut rules = vec![
Some(GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
}),
None,
Some(GeneratedParserRule {
rule_index: 2,
entry_state: 10,
left_recursive: false,
steps: vec![mt(1, 0)],
}),
];
drop_rules_calling_disabled_rules(&mut rules);
assert!(rules[0].is_none());
assert!(rules[1].is_none());
assert!(rules[2].is_some());
}
#[test]
fn generated_parent_keeps_interpreted_child_call() {
let rules = vec![
Some(GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
}),
None,
];
let rendered =
render_generated_rule_dispatch(&rules, &[true, false], &BTreeMap::new(), false);
assert!(rendered.contains(
"0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("parse_generated_rule_1_dispatch"));
}
#[test]
fn classifies_expensive_long_leading_call_chains_as_atn_preferred() {
let mut rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
assert_eq!(
generated_atn_preferred_rule_calls(&rules, &[]),
vec![true; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN]
);
let required = BTreeSet::from([ATN_PREFERRED_LEADING_CALL_CHAIN_MIN - 1]);
assert_eq!(
generated_atn_preferred_rule_calls_excluding(
&rules,
&[],
&generated_rule_callers_reaching(&rules, &required),
),
vec![false; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN],
"portable-local owners and generated callers must stay on the generated path"
);
rules.truncate(ATN_PREFERRED_LEADING_CALL_CHAIN_MIN - 1);
assert_eq!(
generated_atn_preferred_rule_calls(&rules, &[]),
vec![false; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN - 1]
);
}
#[test]
fn atn_preferred_rule_calls_reject_simple_operator_ladders() {
let simple_rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let steps = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
vec![adaptive_loop(rule_index), mt(1, 0)]
} else {
vec![cr(rule_index + 1), adaptive_loop(rule_index)]
};
Some(test_rule(rule_index, steps))
})
.collect::<Vec<_>>();
assert_eq!(
generated_atn_preferred_rule_calls(&simple_rules, &[]),
vec![false; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN]
);
let expensive_rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
assert_eq!(
generated_atn_preferred_rule_calls(&expensive_rules, &[]),
vec![true; ATN_PREFERRED_LEADING_CALL_CHAIN_MIN]
);
}
#[test]
fn atn_preferred_rule_calls_propagate_through_expensive_wrappers() {
let mut rules = Vec::new();
rules.push(Some(test_rule(
0,
vec![mt(9, 0), adaptive_loop(100), adaptive_loop(101), cr(1)],
)));
rules.push(Some(test_rule(
1,
vec![mt(8, 0), adaptive_loop(102), adaptive_loop(103), cr(2)],
)));
for rule_index in 2..(2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN) {
let next = if rule_index + 1 == 2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
rules.push(Some(expensive_ladder_rule(rule_index, next)));
}
rules.push(Some(test_rule(10, vec![cr(2)])));
let mut expected = vec![true; 2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN];
expected.push(false);
assert_eq!(generated_atn_preferred_rule_calls(&rules, &[]), expected);
}
#[test]
fn renders_atn_preferred_generated_child_calls_as_interpreted_by_default() {
let rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
let direct_generated_rule_calls = vec![true; rules.len()];
let rule_names = Vec::new();
let rendered = render_generated_rule_dispatch_with_rule_names(
&rules,
&direct_generated_rule_calls,
&rule_names,
&BTreeMap::new(),
true,
None,
None,
);
// ATN-preferred children route through `parse_rule_precedence_from_generated`:
// the rule's generated dispatch arm is `generated_only()`-guarded, so in normal
// mode the wrapper parses the child interpreted (optimization preserved) while
// buffering its actions in position (correct ordering).
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(1, 0)"));
}
#[test]
fn renders_atn_preferred_dispatch_only_for_generated_only_mode() {
let mut rules = Vec::new();
rules.push(Some(test_rule(
0,
vec![mt(9, 0), adaptive_loop(100), adaptive_loop(101), cr(2)],
)));
rules.push(Some(test_rule(1, vec![mt(1, 0)])));
for rule_index in 2..(2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN) {
let next = if rule_index + 1 == 2 + ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
rules.push(Some(expensive_ladder_rule(rule_index, next)));
}
let direct_generated_rule_calls = vec![true; rules.len()];
let rule_names = Vec::new();
let rendered = render_generated_rule_dispatch_with_rule_names(
&rules,
&direct_generated_rule_calls,
&rule_names,
&BTreeMap::new(),
true,
None,
None,
);
assert!(rendered.contains(
"0 if self.generated_only() => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
assert!(!rendered.contains(
"0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
// The ATN-preferred child call routes through the buffering wrapper.
assert!(rendered.contains("self.parse_rule_precedence_from_generated(2, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(2, 0)"));
}
#[test]
fn embedded_rules_never_use_atn_preferred_fallback() {
let rules = (0..ATN_PREFERRED_LEADING_CALL_CHAIN_MIN)
.map(|rule_index| {
let next = if rule_index + 1 == ATN_PREFERRED_LEADING_CALL_CHAIN_MIN {
None
} else {
Some(rule_index + 1)
};
Some(expensive_ladder_rule(rule_index, next))
})
.collect::<Vec<_>>();
let direct_generated_rule_calls = vec![true; rules.len()];
let adaptive_decisions = BTreeSet::new();
let ll1_decision_arms = BTreeMap::new();
let predicates = BTreeMap::new();
let rule_has_attrs = vec![false; rules.len()];
let init_entry = BTreeMap::new();
let after = BTreeMap::new();
let call_args = BTreeMap::new();
let rule_arg0 = vec![None; rules.len()];
let rendered = render_generated_rule_dispatch_with_rule_names(
&rules,
&direct_generated_rule_calls,
&[],
&BTreeMap::new(),
true,
Some(EmbeddedStepRender {
force_adaptive: false,
adaptive_decisions: &adaptive_decisions,
ll1_decision_arms: &ll1_decision_arms,
predicates: &predicates,
rule_has_attrs: &rule_has_attrs,
init_entry: &init_entry,
after: &after,
call_args: &call_args,
rule_arg0: &rule_arg0,
}),
None,
);
assert!(!rendered.contains("if self.generated_only()"));
assert!(rendered.contains(
"0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))"
));
assert!(rendered.contains("self.parse_generated_rule_1_dispatch(0, false)"));
assert!(!rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
}
#[test]
fn compiles_token_set_transitions() {
let range_atn = transition_atn(|_| ParserTransitionSpec::Range {
target: 7,
start: 2,
stop: 4,
});
let range = only_transition(&range_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
range,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(ms(vec![(2, 4)], 7)), 7))
);
let set_atn = transition_atn(|builder| {
let set = builder.add_interval_set([(1, 1), (5, 6)]).expect("set");
ParserTransitionSpec::Set { target: 8, set }
});
let set_transition = only_transition(&set_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
set_transition,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(ms(vec![(1, 1), (5, 6)], 8)), 8))
);
let not_set_atn = transition_atn(|builder| {
let set = builder.add_interval_set([(1, 1)]).expect("set");
ParserTransitionSpec::NotSet { target: 9, set }
});
let not_set_transition = only_transition(¬_set_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
not_set_transition,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(mns(vec![(1, 1)], 9)), 9))
);
let dense_ranges = (1..=256)
.step_by(2)
.map(|token| (token, token))
.collect::<Vec<_>>();
let dense_set_atn = transition_atn(|builder| {
let set = builder
.add_interval_set(dense_ranges.iter().copied())
.expect("dense set");
ParserTransitionSpec::Set { target: 8, set }
});
let dense_set_transition = only_transition(&dense_set_atn);
assert_eq!(
compile_generated_parser_transition(
3,
&[],
dense_set_transition,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((Some(mts(0, dense_ranges, 8)), 8))
);
}
#[test]
fn compiles_generated_action_transitions_only_for_allowed_states() {
let action_atn = transition_atn(|_| ParserTransitionSpec::Action {
target: 8,
rule_index: 2,
action_index: Some(0),
context_dependent: false,
});
let action = only_transition(&action_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
None
);
let mut generated_action_states = BTreeSet::new();
generated_action_states.insert(4);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &BTreeSet::new(),
generated: &generated_action_states,
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((
Some(GeneratedParserStep::Action {
source_state: 4,
rule_index: 2,
}),
8
))
);
}
#[test]
fn compiles_rule_call_precedence_from_rule_args() {
let rule_atn = transition_atn(|_| ParserTransitionSpec::Rule {
target: 1,
rule_index: 2,
follow_state: 8,
precedence: 0,
});
let rule = only_transition(&rule_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[(4, 2, RuleArgTemplate::Literal(6))],
rule,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((
Some(GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 2,
precedence: GeneratedRuleCallPrecedence::Literal(6),
}),
8
))
);
assert_eq!(
compile_generated_parser_transition(
4,
&[(4, 2, RuleArgTemplate::InheritLocal)],
rule,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((
Some(GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 2,
precedence: GeneratedRuleCallPrecedence::InheritLocal,
}),
8
))
);
}
#[test]
fn parses_boolean_literal_rule_arguments() {
let data = InterpData {
rule_names: vec!["s".to_owned(), "flag".to_owned()],
..InterpData::default()
};
assert_eq!(
literal_rule_arg_calls(
&data,
"parser grammar T;\ns : flag[true] flag[false] ;\nflag[boolean enabled] : ;"
),
[
(1, RuleArgTemplate::Literal(1)),
(1, RuleArgTemplate::Literal(0)),
]
);
}
#[test]
fn boolean_literal_rule_arguments_ignore_comments_literals_and_actions() {
let data = InterpData {
rule_names: vec!["s".to_owned(), "flag".to_owned()],
..InterpData::default()
};
assert_eq!(
literal_rule_arg_calls(
&data,
"parser grammar T;\n\
s : /* flag[true] */ flag[false] 'flag[true]' \"flag[false]\" \
{ flag[true] } ;\n\
flag[boolean enabled] : ;"
),
[(1, RuleArgTemplate::Literal(0))]
);
}
#[test]
fn compiles_synthetic_noop_action_transitions_as_epsilon() {
let action_atn = transition_atn(|_| ParserTransitionSpec::Action {
target: 8,
rule_index: 2,
action_index: None,
context_dependent: false,
});
let action = only_transition(&action_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((None, 8))
);
}
#[test]
fn rejects_known_non_inline_noop_action_transitions() {
let action_atn = transition_atn(|_| ParserTransitionSpec::Action {
target: 8,
rule_index: 2,
action_index: None,
context_dependent: false,
});
let action = only_transition(&action_atn);
let mut action_states = BTreeSet::new();
action_states.insert(4);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
action,
ActionStateSets {
all: &action_states,
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
None
);
}
#[test]
fn compiles_parser_predicates_as_viable_when_no_metadata_is_active() {
let predicate_atn = transition_atn(|_| ParserTransitionSpec::Predicate {
target: 8,
rule_index: 2,
pred_index: 1,
context_dependent: false,
});
let predicate = only_transition(&predicate_atn);
assert_eq!(
compile_generated_parser_transition(
4,
&[],
predicate,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
}
),
Some((None, 8))
);
}
#[test]
fn compiles_generated_parser_predicate_transitions() {
let predicate_atn = transition_atn(|_| ParserTransitionSpec::Predicate {
target: 8,
rule_index: 2,
pred_index: 1,
context_dependent: false,
});
let predicate = only_transition(&predicate_atn);
let mut predicates = BTreeSet::new();
predicates.insert((2, 1));
let generated_predicates = predicates.clone();
assert_eq!(
compile_generated_parser_transition(
4,
&[],
predicate,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &predicates,
generated: &generated_predicates,
}
),
Some((
Some(GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 1,
}),
8
))
);
}
#[test]
fn renders_fail_option_parser_predicate_error() {
let mut rendered = String::new();
render_generated_step(
&mut rendered,
&GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 1,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(
rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 2, 1, &__ctx, __precedence)"
)
);
assert!(rendered.contains("failed_predicate_option_error(2, __message)"));
assert!(rendered.contains("failed_predicate_error(\"semantic predicate\")"));
}
fn render_call_rule_step(
direct_generated_rule_calls: &[bool],
atn_preferred_rule_calls: &[bool],
) -> String {
let mut rendered = String::new();
render_generated_step(
&mut rendered,
&GeneratedParserStep::CallRule {
source_state: 4,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
},
2,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls,
atn_preferred_rule_calls,
},
);
rendered
}
#[test]
fn atn_preferred_child_with_after_action_routes_through_dispatch_wrapper() {
// An ATN-preferred child that carries an `@after` action
// (direct_generated_rule_calls[1] == false) must go through
// `parse_rule_precedence_from_generated`, which preserves interpreted routing
// (the rule's generated dispatch arm is guarded by `generated_only()`) while
// BUFFERING the child's body actions and `@after` in position.
let rendered = render_call_rule_step(&[true, false], &[true, true]);
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(1, 0)"));
}
#[test]
fn atn_preferred_child_without_after_also_routes_through_dispatch_wrapper() {
// An ATN-preferred child WITHOUT `@after` (direct_generated_rule_calls[1] ==
// true) must ALSO route through `parse_rule_precedence_from_generated`, not the
// bare `parse_interpreted_rule_precedence`: the bare interpreted call runs the
// child's body actions immediately, which reorders them before the generated
// parent's buffered actions. The wrapper buffers them in position instead while
// still parsing the child interpreted (the dispatch arm is `generated_only()`
// guarded).
let rendered = render_call_rule_step(&[true, true], &[true, true]);
assert!(rendered.contains("self.parse_rule_precedence_from_generated(1, 0)"));
assert!(!rendered.contains("self.parse_interpreted_rule_precedence(1, 0)"));
}
#[test]
fn rejects_known_parser_predicates_without_generated_metadata() {
let predicate_atn = transition_atn(|_| ParserTransitionSpec::Predicate {
target: 8,
rule_index: 2,
pred_index: 1,
context_dependent: false,
});
let predicate = only_transition(&predicate_atn);
let mut predicates = BTreeSet::new();
predicates.insert((2, 1));
assert_eq!(
compile_generated_parser_transition(
4,
&[],
predicate,
ActionStateSets {
all: &BTreeSet::new(),
generated: &BTreeSet::new(),
inline: &BTreeSet::new(),
},
PredicateCoordinateSets {
all: &predicates,
generated: &BTreeSet::new(),
}
),
None
);
}
#[test]
fn parse_rule_fallback_runs_parser_actions() {
let fallback = render_parser_parse_rule_fallback(false, &[], &[], true, false, None);
assert!(fallback.contains(
"parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence"
));
assert!(fallback.contains("for action in actions { self.run_action(action, tree); }"));
assert!(fallback.contains("Ok(tree)"));
}
#[test]
fn parser_action_dispatch_falls_back_to_semantic_hook() {
let method = render_parser_action_method(true, &BTreeSet::new());
assert!(method.contains("fn run_action"));
assert!(method.contains("self.base.parser_action_hook(action, tree)"));
}
#[test]
fn parser_action_assume_override_gets_explicit_noop_arm() {
// An `assume-*` action override drops its translated arm, but must NOT
// fall through to the `parser_action_hook` catch-all — that would fail
// loud under the Error policy (NoSemanticHooks) or run a user side
// effect for a coordinate the manifest reports as a no-op fallback. It
// gets an explicit empty arm instead. A `hook`/`error` override is not
// in this set, so it still falls through to the hook.
let mut assume_noop = BTreeSet::new();
assume_noop.insert(7_usize);
let method = render_parser_action_method(true, &assume_noop);
// The assume-* state has its own empty arm, placed before the catch-all.
let noop_at = method
.find("7 => {}")
.expect("assume-* action state gets an explicit no-op arm");
let hook_at = method
.find("self.base.parser_action_hook(action, tree)")
.expect("the hook catch-all is still emitted for hook/unknown states");
assert!(
noop_at < hook_at,
"the assume-* no-op arm must precede the hook catch-all"
);
}
#[test]
fn parser_st_actions_do_not_emit_replay_machinery() {
let rendered = render_parser(
"TParser",
&minimal_parser_data(),
Some(r#"parser grammar T; s @after {<InputText():writeln()>} : ;"#),
)
.expect("parser should render");
for removed in [
format!("{}{}", "Generated", "Action"),
format!("{}{}", "generated", "_actions"),
format!("{}{}", "Member", "Snapshot"),
format!("{}{}", "run_after", "_actions"),
format!("{}{}", "int_members", "_checkpoint"),
format!("{}{}", "restore_int", "_members"),
format!("{}{}", "CTX_ROOTED", "_ACTION_STATES"),
] {
assert!(
!rendered.contains(&removed),
"removed parser replay machinery leaked into generated module: {removed}"
);
}
}
#[test]
fn embedded_init_action_runs_at_rule_entry() {
let rendered = render_parser_with_options(
"TParser",
&minimal_parser_data(),
Some(r#"parser grammar T; s @init {println!("init");} : ;"#),
ParserRenderOptions {
embedded: true,
..ParserRenderOptions::default()
},
)
.expect("embedded parser should render");
let start_at = rendered
.find("let __rule_start")
.expect("generated rule captures rule start");
let init_at = rendered
.find("println!(\"init\");")
.expect("embedded @init body is emitted");
let body_at = rendered
.find("let mut __consumed_eof")
.expect("generated rule body follows entry setup");
assert!(
start_at < init_at && init_at < body_at,
"embedded @init must run after rule entry setup and before the rule body"
);
}
#[test]
fn embedded_listener_forwards_error_nodes() {
let rendered = render_parser_with_options(
"TParser",
&minimal_parser_data(),
Some("parser grammar T; s : ;"),
ParserRenderOptions {
embedded: true,
..ParserRenderOptions::default()
},
)
.expect("embedded parser should render");
assert!(rendered.contains("ErrorNodeView as RuntimeErrorNode"));
assert!(rendered.contains("pub struct ErrorNode<'a>"));
assert!(rendered.contains("fn visit_error_node(&mut self, _node: &ErrorNode)"));
assert!(
rendered
.contains("fn visit_error_node(&mut self, node: RuntimeErrorNode<'_>) -> Result<")
);
assert!(rendered.contains("self.0.visit_error_node(&ErrorNode::new(node));"));
}
#[test]
fn embedded_active_context_preserves_invocation_states() {
let rendered = render_parser_with_options(
"TParser",
&minimal_parser_data(),
Some("parser grammar T; s : ;"),
ParserRenderOptions {
embedded: true,
..ParserRenderOptions::default()
},
)
.expect("embedded parser should render");
assert!(rendered.contains("invocation_states: Vec<isize>"));
assert!(rendered.contains("__invocation_states: invocation_states"));
assert!(rendered.contains("::__from_child_node(node, &self.__invocation_states)"));
assert!(rendered.contains("::__from_listener_node(context, self.1.as_deref())"));
assert!(rendered.contains("pub fn walk_with_invocation_states"));
assert!(
!rendered.contains("__GeneratedRuleContext::Active { .. } => Vec::new()"),
"active contexts must preserve their invoking-state chain"
);
}
#[test]
fn non_embedded_parser_action_disables_generated_rule() {
let rendered = render_parser(
"TParser",
&action_parser_data(),
Some(r#"parser grammar T; s : {<writeln("x")>} A ;"#),
)
.expect("parser should render");
assert!(
!rendered.contains("parse_generated_rule_0_dispatch"),
"non-embedded parser action rules must stay on the interpreted path"
);
assert!(rendered.contains("self.base.parser_action_hook(action, tree)"));
assert!(!rendered.contains(&format!("{}{}", "Generated", "Action")));
assert!(!rendered.contains(&format!("{}{}", "generated", "_actions")));
}
#[test]
fn synthetic_parser_action_does_not_disable_generated_rule() {
let rendered = render_parser(
"TParser",
&action_parser_data(),
Some(r#"parser grammar T; s : A ;"#),
)
.expect("parser should render");
assert!(
rendered.contains("parse_generated_rule_0_dispatch"),
"a synthetic/no-op ATN action state must not disable generated parsing"
);
}
#[test]
fn context_superclass_does_not_disable_generated_rules() {
let rendered = render_parser(
"TParser",
&minimal_parser_data(),
Some(
r#"parser grammar T;
options { contextSuperClass=MyRuleNode; }
<TreeNodeWithAltNumField(X="T")>
s : ;
"#,
),
)
.expect("parser should render");
assert!(rendered.contains("parse_generated_rule_0"));
assert!(rendered.contains("track_alt_numbers: true"));
}
#[test]
fn generated_parser_handles_diagnostic_reporting() {
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(!rendered.contains("if !self.base.report_diagnostic_errors() || __generated_only"));
assert!(
rendered.contains("self.parse_interpreted_rule_precedence(rule_index, precedence)?")
);
}
#[test]
fn generated_only_mode_disables_missing_rule_fallback() {
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(rendered.contains("ANTLR4_RUST_GENERATED_ONLY"));
assert!(rendered.contains("let __generated_only = self.generated_only();"));
assert!(!rendered.contains("GeneratedRuleError::Recoverable"));
assert!(rendered.contains("generated parser did not emit rule {}"));
}
#[test]
fn require_generated_parser_reports_missing_rules() {
let error = require_all_parser_rules_generated(&[None], &minimal_parser_data())
.expect_err("missing generated rule should fail strict mode");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(
error.to_string(),
"generated parser did not emit 1 rule(s): s"
);
}
#[test]
fn portable_local_semantics_reject_missing_generated_owner() {
let error = require_portable_local_rules_generated(
&[None],
&BTreeSet::from([0]),
&minimal_parser_data(),
)
.expect_err("portable local semantics cannot use interpreted fallback");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(
error.to_string(),
"portable local semantics require 1 generated parser rule(s): s"
);
}
#[test]
fn portable_local_semantics_reject_missing_generated_caller() {
let required = atn_rule_callers_reaching(&entry_candidate_atn(), &BTreeSet::from([1]), 4);
assert_eq!(required, BTreeSet::from([0, 1, 2]));
let rules = vec![
None,
Some(test_rule(1, Vec::new())),
Some(test_rule(2, Vec::new())),
Some(test_rule(3, Vec::new())),
];
let data = InterpData {
rule_names: vec![
"firstEntry".to_owned(),
"child".to_owned(),
"secondEntry".to_owned(),
"recursive".to_owned(),
],
..InterpData::default()
};
let error = require_portable_local_rules_generated(&rules, &required, &data)
.expect_err("interpreted callers cannot bypass generated local state");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(
error.to_string(),
"portable local semantics require 1 generated parser rule(s): firstEntry"
);
}
#[test]
fn renders_parse_convenience_without_replacing_manual_constructor() {
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(rendered.contains("pub struct TParserParseOutput<R, L>"));
assert!(rendered.contains("pub result: R,"));
assert!(rendered.contains("pub parser: TParser<L>,"));
assert!(rendered.contains("pub fn parse<L: TokenSource>("));
assert!(rendered.contains("pub fn parse_with_parser<L: TokenSource, R>("));
assert!(
!rendered
.contains(") -> Result<R, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource,")
);
assert!(!rendered.contains(
") -> Result<TParserParseOutput<R, L>, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource,"
));
assert!(rendered.contains("lexer: impl FnOnce(antlr4_runtime::InputStream) -> L"));
assert!(rendered.contains("antlr4_runtime::InputStream::new(input.as_ref())"));
assert!(rendered.contains("let tokens = CommonTokenStream::new(lexer);"));
assert!(rendered.contains("let result = entry(&mut parser)?;"));
assert!(rendered.contains("Ok(TParserParseOutput { result, parser })"));
assert!(rendered.contains(
"let TParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?;"
));
assert!(rendered.contains("Ok(parser.into_parsed_file(result))"));
assert!(rendered.contains("pub fn new(input: CommonTokenStream<L>) -> Self"));
assert!(
rendered.contains("pub fn with_hooks(input: CommonTokenStream<L>, hooks: H) -> Self")
);
}
#[test]
fn generated_parse_output_name_does_not_collide_with_parser_type() {
let rendered = render_parser("ParseOutput", &minimal_parser_data(), None)
.expect("parser should render");
assert!(rendered.contains("pub struct ParseOutputParseOutput<R, L>"));
assert!(rendered.contains("pub parser: ParseOutput<L>,"));
assert!(
rendered
.contains(") -> Result<ParseOutputParseOutput<R, L>, antlr4_runtime::AntlrError>")
);
assert!(rendered.contains("Ok(ParseOutputParseOutput { result, parser })"));
}
#[test]
fn generated_parser_reports_lexer_errors_on_outer_success() {
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(rendered.contains("if allow_generated_fallback {"));
assert!(rendered.contains("self.base.report_generated_parser_diagnostics();"));
assert!(rendered.contains("fn number_of_syntax_errors(&self) -> usize"));
assert!(!rendered.contains("self.base.report_token_source_errors();"));
}
#[test]
fn generated_parser_exposes_owned_token_stream() {
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
assert!(rendered.contains("pub const fn token_stream(&self) -> &CommonTokenStream<L>"));
assert!(rendered.contains("self.base.token_stream()"));
assert!(
rendered
.contains("pub const fn token_stream_mut(&mut self) -> &mut CommonTokenStream<L>")
);
assert!(rendered.contains("self.base.token_stream_mut()"));
assert!(
rendered.contains("pub fn set_token_stream(&mut self, input: CommonTokenStream<L>)")
);
assert!(rendered.contains("self.base.set_token_stream(input)"));
assert!(rendered.contains("pub fn reset(&mut self)"));
assert!(rendered.contains("self.base.reset()"));
assert!(rendered.contains("pub fn clear_dfa(&mut self)"));
assert!(rendered.contains("simulator.clear_dfa()"));
assert!(rendered.contains("ParserAtnSimulator::clear_shared_dfa(atn())"));
assert!(rendered.contains("pub fn add_error_listener<T>(&mut self, listener: T)"));
assert!(rendered.contains(
"T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,"
));
assert!(rendered.contains("self.base.add_error_listener(listener)"));
assert!(rendered.contains("pub fn remove_error_listeners(&mut self)"));
assert!(rendered.contains("self.base.remove_error_listeners()"));
assert!(rendered.contains("pub fn into_token_stream(self) -> CommonTokenStream<L>"));
assert!(rendered.contains("self.base.into_token_stream()"));
assert!(
rendered.contains("pub const fn token_store(&self) -> &antlr4_runtime::TokenStore")
);
assert!(rendered.contains("self.base.token_store()"));
assert!(rendered.contains("pub fn into_token_store(self) -> antlr4_runtime::TokenStore"));
assert!(rendered.contains("self.base.into_token_store()"));
}
#[test]
fn generated_parser_renames_rule_wrapper_that_collides_with_token_stream_accessor() {
let mut data = minimal_parser_data();
data.rule_names = vec!["tokenStream".to_owned()];
let rendered = render_parser("TParser", &data, None).expect("parser should render");
assert!(rendered.contains("pub const fn token_stream(&self) -> &CommonTokenStream<L>"));
assert!(rendered.contains(
"pub fn token_stream_rule(&mut self) -> Result<antlr4_runtime::ParseTree, antlr4_runtime::AntlrError>"
));
assert!(rendered.contains("/// - `token_stream_rule()`"));
assert!(!rendered.contains("/// - `token_stream()`"));
assert!(!rendered.contains("pub fn token_stream(&mut self)"));
}
#[test]
fn generated_rule_recovers_own_sync_failure_unless_top_level() {
// A rule's own sync failure (`__sync_error`) is fatal only at the top-level
// public entry (`allow_fallback`); a nested child recovers it locally and
// returns a partial subtree (so a parent never recovers the child's failure
// on the parent context, losing the child subtree). Assert the generated
// catch arm gates the `Fatal` return on `allow_fallback` and otherwise runs
// `recover_generated_rule` + `finish_rule` + `Ok`.
let rendered =
render_parser("TParser", &minimal_parser_data(), None).expect("parser should render");
let sync_arm = rendered
.find("if let Some(__error) = __sync_error {")
.expect("sync-error catch arm present");
let rest = &rendered[sync_arm..];
// Inside the sync arm, the Fatal return is guarded by `if allow_fallback`.
let guard = rest
.find("if allow_fallback {")
.expect("fatal return gated on allow_fallback");
let fatal = rest
.find("return Err(GeneratedRuleError::Fatal(__error));")
.expect("fatal return present");
assert!(
guard < fatal,
"Fatal return must be inside the allow_fallback guard"
);
let count = rest
.find("self.base.record_generated_syntax_error();")
.expect("fatal sync path records syntax error");
assert!(
guard < count && count < fatal,
"fatal sync path must increment before returning"
);
// And the nested-child path recovers locally and returns Ok.
let recover = rest
.find("self.base.recover_generated_rule(&mut __ctx, atn(), __error);")
.expect("local recovery present in sync arm");
assert!(
recover > guard,
"recover path follows the guarded fatal return"
);
assert!(rest[recover..].contains("return Ok(__tree);"));
}
#[test]
fn call_rule_step_skips_child_action_scaffolding_without_parser_actions() {
let rendered = render_call_rule_step(&[true, true], &[false, false]);
assert!(rendered.contains("let __child = self.parse_generated_rule_1_dispatch(0, false).map_err(GeneratedRuleError::into_error);"));
assert!(rendered.contains("self.base.discard_invoking_state(__invoking_marker);"));
assert!(rendered.contains("let __child = __child?;"));
assert!(rendered.contains("self.base.add_parse_child(&mut __ctx, __child);"));
assert!(!rendered.contains("__child_action_marker"));
assert!(!rendered.contains("__child_member_checkpoint"));
assert!(!rendered.contains(&format!(
"{}{}{}{}",
"Generated", "Action::", "Member", "Snapshot"
)));
assert!(!rendered.contains(&format!(
"{}{}{}",
"CTX_ROOTED", "_ACTION_STATES", ".contains"
)));
}
#[test]
fn renders_wildcard_match_through_recovering_path() {
// A wildcard (`.`) must go through the recovering match (modeled as an
// empty-complement not-set over the full vocabulary) so a wildcard at EOF
// performs ANTLR's single-token insertion instead of aborting the rule.
let rule = GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![GeneratedParserStep::MatchWildcard { follow_state: 7 }],
};
let rendered = render_generated_rule_dispatch(&[Some(rule)], &[], &BTreeMap::new(), false);
// Recovering not-set over 1..=max with an empty exclusion = "any token",
// threading the wildcard's follow state for EOF-insertion follow checks.
assert!(
rendered.contains("match_not_set_recovering(&[], 1, atn().max_token_type(), 7, atn())")
);
assert!(rendered.contains("__consumed_eof |= __match.consumed_eof();"));
// The old non-recovering call must be gone.
assert!(!rendered.contains("self.base.match_wildcard()"));
}
#[test]
fn renders_packed_token_sets_for_generated_matches_and_lookahead() {
let step = mts(4, vec![(2, 4), (9, 9)], 7);
let rule = GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![step.clone()],
};
let rendered = render_generated_rule_dispatch(&[Some(rule)], &[], &BTreeMap::new(), false);
assert!(rendered.contains(
"match_token_set_recovering(atn().token_set(4).expect(\"generated parser token-set index\"), 7, atn())"
));
assert_eq!(
leading_lookahead_condition(&[step], "__la"),
Some(
"atn().token_set(4).expect(\"generated parser token-set index\").contains(__la)"
.to_owned()
)
);
let not_step = mnts(5, vec![(3, 6)], 8);
let not_rule = GeneratedParserRule {
rule_index: 0,
entry_state: 0,
left_recursive: false,
steps: vec![not_step.clone()],
};
let rendered =
render_generated_rule_dispatch(&[Some(not_rule)], &[], &BTreeMap::new(), false);
assert!(rendered.contains(
"match_not_token_set_recovering(atn().token_set(5).expect(\"generated parser token-set index\"), 1, atn().max_token_type(), 8, atn())"
));
assert_eq!(
leading_lookahead_condition(&[not_step], "__la"),
Some(
"(1..=atn().max_token_type()).contains(&__la) && !(atn().token_set(5).expect(\"generated parser token-set index\").contains(__la))"
.to_owned()
)
);
}
#[test]
fn generated_decision_does_not_reject_semantic_context_metadata() {
let alts = vec![vec![mt(1, 0)], vec![]];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("ll1_decision_prediction(atn(), 1)"));
assert!(rendered.contains("prediction_mode() != antlr4_runtime::PredictionMode::Sll"));
assert!(!rendered.contains("has_semantic_context"));
}
#[test]
fn generated_decision_filters_semantic_predicate_alts() {
let alts = vec![
vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(1, 2),
],
vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 1,
},
mt(1, 3),
],
vec![mt(2, 4)],
];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("if __prediction.has_semantic_context"));
assert!(rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 1, 0, &__ctx, __precedence)"
));
assert!(rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 1, 1, &__ctx, __precedence)"
));
assert!(rendered.contains("__semantic_la == 1"));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: __alt, ..__prediction }")
);
assert!(rendered.contains("no_viable_alternative_error(__decision_start)"));
assert!(!rendered.contains("__sync_error = Some(__error.clone())"));
}
#[test]
fn generated_decision_does_not_hoist_portable_predicate_past_local_action() {
let alts = vec![
vec![
GeneratedParserStep::Action {
source_state: 5,
rule_index: 1,
},
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(1, 2),
],
vec![mt(1, 3)],
];
let declarations = vec![
Vec::new(),
vec!["let mut __antlr_local_seen = false;".to_owned()],
];
let inline_actions = BTreeMap::from([(5, "__antlr_local_seen = true;".to_owned())]);
let predicates = BTreeMap::from([((1, 0), ("__antlr_local_seen".to_owned(), None))]);
let required_generated_rules = BTreeSet::from([1]);
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: Some(PortableLocalStepRender {
declarations: &declarations,
inline_actions: &inline_actions,
predicates: &predicates,
required_generated_rules: &required_generated_rules,
}),
inline_action_statements: &inline_actions,
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(!rendered.contains("let __semantic_alt"));
assert!(!rendered.contains("__semantic_la == 1 && (__antlr_local_seen)"));
let assignment = rendered
.find("__antlr_local_seen = true;")
.expect("portable assignment is rendered in the committed alternative");
let predicate = rendered
.find("if !(__antlr_local_seen)")
.expect("portable predicate is evaluated in the committed alternative");
assert!(assignment < predicate);
}
#[test]
fn generated_decision_records_adaptive_diagnostics() {
let alts = vec![vec![mt(1, 4)], vec![mt(2, 5)]];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 16,
decision: 0,
track_alt_number: false,
allow_semantic_context: false,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(
rendered.contains("record_generated_prediction_diagnostic(atn(), 16, &__prediction)")
);
assert!(!rendered.contains("__diagnostic_la"));
}
#[test]
fn generated_semantic_decision_reports_filtered_ambiguity_diagnostics() {
let alts = vec![
vec![mt(2, 4)],
vec![mt(2, 5)],
vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(2, 6),
],
];
let mut rendered = String::new();
render_generated_decision(
&mut rendered,
DecisionRender {
state: 16,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: &alts,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("if self.base.report_diagnostic_errors()"));
assert!(rendered.contains("let __diagnostic_la = self.base.la(1);"));
assert!(rendered.contains("if __diagnostic_la == 2"));
assert!(rendered.contains("__diagnostic_alts.push(1);"));
assert!(rendered.contains("__diagnostic_alts.push(2);"));
assert!(rendered.contains(
"record_generated_ambiguity_diagnostic(atn(), 16, __decision_start, __decision_start, &__diagnostic_alts)"
));
}
#[test]
fn generated_loop_filters_failed_leading_predicate_to_exit_alt() {
let body = vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(3, 4),
];
let mut rendered = String::new();
render_generated_star_loop(
&mut rendered,
StarLoopRender {
state: 1,
decision: 0,
alts: (1, 2),
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
plus_loop: false,
fast_path: None,
body: &body,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("if __prediction.alt == 1"));
assert!(rendered.contains(
"parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 1, 0, &__ctx, __precedence)"
));
assert!(rendered.contains("__semantic_la == 3"));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: 2, ..__prediction }")
);
}
#[test]
fn generated_loop_filters_portable_local_predicate() {
let body = vec![
GeneratedParserStep::Predicate {
rule_index: 1,
pred_index: 0,
},
mt(3, 4),
];
let declarations = vec![vec!["let mut __antlr_local_seen = false;".to_owned()]];
let predicates = BTreeMap::from([((1, 0), ("__antlr_local_seen".to_owned(), None))]);
let required_generated_rules = BTreeSet::from([1]);
let mut rendered = String::new();
render_generated_star_loop(
&mut rendered,
StarLoopRender {
state: 1,
decision: 0,
alts: (1, 2),
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
plus_loop: false,
fast_path: None,
body: &body,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: Some(PortableLocalStepRender {
declarations: &declarations,
inline_actions: &BTreeMap::new(),
predicates: &predicates,
required_generated_rules: &required_generated_rules,
}),
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("__semantic_la == 3 && (__antlr_local_seen)"));
assert!(!rendered.contains("parser_semantic_ir_predicate_matches"));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: 2, ..__prediction }")
);
}
#[test]
fn generated_loop_filters_first_nested_predicated_decision() {
let body = vec![GeneratedParserStep::Decision {
state: 1,
decision: 0,
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
fast_path: None,
alts: vec![
vec![mt(1, 4)],
vec![mt(3, 4)],
vec![
GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 0,
},
mt(2, 4),
],
],
}];
let mut rendered = String::new();
render_generated_star_loop(
&mut rendered,
StarLoopRender {
state: 1,
decision: 1,
alts: (1, 2),
track_alt_number: false,
allow_semantic_context: true,
force_context: false,
plus_loop: false,
fast_path: None,
body: &body,
},
0,
GeneratedStepRenderContext {
embedded: None,
portable_locals: None,
inline_action_statements: &BTreeMap::new(),
track_alt_numbers: false,
direct_generated_rule_calls: &[],
atn_preferred_rule_calls: &[],
},
);
assert!(rendered.contains("(__semantic_la == 1) || (__semantic_la == 3)"));
// The lookahead guard comes first so `&&` short-circuits on it before the
// predicate hook runs; a non-matching lookahead must not trigger a
// fail-loud hit for an unknown/hook predicate on a non-candidate alt.
assert!(rendered.contains(
"(__semantic_la == 2 && self.base.parser_semantic_ir_predicate_matches_with_context_and_local(parser_semantics(), 2, 0, &__ctx, __precedence))"
));
assert!(
rendered.contains("antlr4_runtime::ParserAtnPrediction { alt: 2, ..__prediction }")
);
}
#[test]
fn semantic_candidate_condition_guards_predicate_behind_lookahead() {
// A leading predicate followed by a token match must render as
// `lookahead && predicate`, so `&&` short-circuits on the side-effect-free
// lookahead before invoking the predicate hook. Otherwise an alternative
// whose first token cannot match the current lookahead would still
// evaluate its hook/unknown predicate, recording a spurious fail-loud
// `Unsupported` hit under `--sem-unknown=hook`/`error` and rejecting a
// later syntactically viable alternative.
let steps = vec![
GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 0,
},
mt(7, 4),
];
let condition = semantic_alt_candidate_condition(&steps, None, None);
let la_at = condition
.find("__semantic_la == 7")
.expect("condition includes the leading lookahead guard");
let pred_at = condition
.find("parser_semantic_ir_predicate_matches_with_context_and_local")
.expect("condition includes the leading predicate");
assert!(
la_at < pred_at,
"lookahead must be evaluated before the predicate hook: {condition}"
);
assert!(
condition.starts_with("__semantic_la == 7 &&"),
"the lookahead guard must be the first `&&` operand: {condition}"
);
}
#[test]
fn semantic_alt_guard_classifies_unresolved_rule_call_alt() {
// An alt whose first consuming step is a rule call, with no leading
// predicate or lookahead, is "unresolved" (FIRST set not computed here).
let rule_call = vec![GeneratedParserStep::CallRule {
source_state: 5,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}];
assert!(semantic_alt_guard_is_unresolved(&rule_call, None));
// A token-led alt is resolved (concrete lookahead), so NOT unresolved.
assert!(!semantic_alt_guard_is_unresolved(&[mt(7, 4)], None));
// A predicate-led alt is resolved (guarded by the predicate).
assert!(!semantic_alt_guard_is_unresolved(
&[GeneratedParserStep::Predicate {
rule_index: 2,
pred_index: 0,
}],
None,
));
// A pure epsilon alt (no consuming step) legitimately matches; not unresolved.
assert!(!semantic_alt_guard_is_unresolved(&[], None));
}
#[test]
fn semantic_alt_search_orders_unresolved_alts_last() {
// `{p()}? 'a' | x | 'a'` (alt 2 = rule call `x`): when alt 1's predicate
// fails, the two-pass search tries resolved-guard alts first (alt 3's
// concrete lookahead), then the unresolved rule-call alt as a last
// resort. So alt 3 is NOT shadowed by alt 2, AND alt 2 stays reachable.
let alts = vec![
vec![
GeneratedParserStep::Predicate {
rule_index: 0,
pred_index: 0,
},
mt(1, 4),
],
vec![GeneratedParserStep::CallRule {
source_state: 5,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
vec![mt(1, 4)],
];
let alt_conditions = alts
.iter()
.map(|steps| semantic_alt_candidate_condition(steps, None, None))
.collect::<Vec<_>>();
let mut rendered = String::new();
render_semantic_alt_search(&mut rendered, "", &alt_conditions, &alts, None);
// The resolved token alt 3 is emitted before the unresolved rule-call
// alt 2 (which keeps its real `true` condition as a last-resort branch),
// so alt 3 wins on a matching lookahead but alt 2 is still reachable.
let alt3_at = rendered
.find("Some(3)")
.expect("resolved token alt 3 is present");
let alt2_at = rendered
.find("Some(2)")
.expect("unresolved rule-call alt 2 is still present (last resort)");
assert!(
alt3_at < alt2_at,
"resolved alt 3 must be tried before the unresolved rule-call alt 2: {rendered}"
);
// Alt 2 is NOT disabled (`if false`); it keeps a reachable branch.
assert!(
!rendered.contains("if false {"),
"unresolved alt must be a last-resort branch, not disabled: {rendered}"
);
assert!(
rendered.contains("if __semantic_la == 1 {\n Some(3)"),
"the token alt keeps its concrete lookahead guard: {rendered}"
);
}
#[test]
fn semantic_alt_search_keeps_lone_unresolved_alt_reachable() {
// `{p()}? 'a' | x` (alt 2 = rule call `x`, the only non-predicated alt):
// when alt 1's predicate fails and no resolved alt matches, the search
// must still try the unresolved alt rather than emitting nothing (which
// would be a spurious NoViableAlt). Codex's counter-example.
let alts = vec![
vec![
GeneratedParserStep::Predicate {
rule_index: 0,
pred_index: 0,
},
mt(1, 4),
],
vec![GeneratedParserStep::CallRule {
source_state: 5,
rule_index: 1,
precedence: GeneratedRuleCallPrecedence::Literal(0),
}],
];
let alt_conditions = alts
.iter()
.map(|steps| semantic_alt_candidate_condition(steps, None, None))
.collect::<Vec<_>>();
let mut rendered = String::new();
render_semantic_alt_search(&mut rendered, "", &alt_conditions, &alts, None);
// The lone unresolved alt is reachable via its real condition, not disabled.
assert!(
rendered.contains("Some(2)") && !rendered.contains("if false {"),
"a lone unresolved alt must remain reachable as a last resort: {rendered}"
);
}
#[test]
fn parses_column_predicate_templates() {
assert_eq!(
parse_predicate_template(r#"<TokenStartColumnEquals("0")>"#),
Some(PredicateTemplate::TokenStartColumnEquals(0))
);
assert_eq!(
parse_predicate_template(r#"<Column()> \< 2"#),
Some(PredicateTemplate::ColumnLessThan(2))
);
assert_eq!(
parse_predicate_template("<Column()> >= 2"),
Some(PredicateTemplate::ColumnGreaterOrEqual(2))
);
}
#[test]
fn extracts_predicate_expression_blocks() {
let templates = extract_supported_predicate_templates(
r#"fragment ID1 : { <Column()> \< 2 }? [a-zA-Z];
fragment ID2 : { <Column()> >= 2 }? [a-zA-Z];"#,
&SemPatternFile::default(),
)
.expect("supported predicate expressions should extract");
assert_eq!(
templates,
[
PredicateTemplate::ColumnLessThan(2),
PredicateTemplate::ColumnGreaterOrEqual(2)
]
);
}
#[test]
fn native_comparison_predicate_falls_through_instead_of_aborting() {
// A native target-language comparison predicate merely contains a `<`
// operator; it is not an ANTLR `<...>` StringTemplate, so it must fall
// through to the unknown-predicate policy (no template) rather than
// aborting codegen with "unsupported target predicate template".
let templates = extract_supported_predicate_templates(
"grammar T;\nr : {this.level < 2}? ID ;\n",
&SemPatternFile::default(),
)
.expect("native comparison predicate must not abort generation");
assert!(
templates.is_empty(),
"native `<` comparison yields no template, deferring to policy: {templates:?}"
);
// A genuine untranslated `<...>` StringTemplate still errors.
let error = extract_supported_predicate_templates(
"grammar T;\nr : {<UnknownTemplate()>}? ID ;\n",
&SemPatternFile::default(),
)
.expect_err("an untranslated <...> StringTemplate predicate must still abort");
assert!(
error
.to_string()
.contains("unsupported target predicate template")
);
}
#[test]
fn predicate_slots_preserve_position_for_mixed_lexers() {
// A translated column predicate followed by a native (untranslated) one
// must yield [Some, None] — the slot walk keeps position so the lexer
// pairs the translated one with its ATN transition and leaves the other
// to the `run_predicate` catch-all, instead of a length-mismatch abort.
let slots = extract_predicate_template_slots_filtered(
"lexer grammar L;\nA : { <Column()> >= 2 }? [a-z]+ ;\nB : {aheadIsDigit()}? [0-9]+ ;\n",
&SemPatternFile::default(),
None,
SemanticsKind::LexerPredicate,
)
.expect("slot extraction succeeds");
assert_eq!(slots.len(), 2, "one slot per predicate block: {slots:?}");
assert_eq!(slots[0], Some(PredicateTemplate::ColumnGreaterOrEqual(2)));
assert_eq!(slots[1], None, "the native predicate stays uncovered");
}
#[test]
fn parser_predicate_scan_skips_lexer_rule_predicates() {
// Combined grammar with a *translatable* lexer-rule predicate preceding
// the parser rule. The parser ATN (predicate_parser_data) has a single
// predicate transition on rule `s`. Without rule-name filtering the
// lexer predicate's template would be paired against `s`'s coordinate
// (mis-mapping) and the leftover would error with "no parser ATN
// predicate transition". The filter must skip the lexer-rule predicate
// so only `s`'s coordinate is considered.
let combined = "grammar S;\nID : { <Column()> >= 2 }? [a-z]+ ;\ns : {isTypeName()}? A ;\n";
let templates = parser_predicate_templates(
&predicate_parser_data(),
combined,
&SemPatternFile::default(),
)
.expect("the lexer-rule predicate must be skipped, not mis-paired or errored");
// `s`'s predicate body (`isTypeName()`) has no built-in template, so
// nothing maps — crucially, the translatable lexer predicate did NOT
// get mapped onto `s`'s coordinate.
assert!(
templates.is_empty(),
"lexer-rule predicate must not map onto a parser coordinate: {templates:?}"
);
}
#[test]
fn parser_predicate_scan_maps_parser_rule_after_lexer_predicate() {
// Same combined shape, but the parser predicate is translatable via a
// coordinate override. Only the parser-rule predicate (rule `s`, pred 0)
// should map, proving the lexer predicate is filtered rather than
// shifting the parser predicate onto the wrong coordinate.
let combined = "grammar S;\nID : { <Column()> >= 2 }? [a-z]+ ;\ns : {isTypeName()}? A ;\n";
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
let templates = parser_predicate_templates(&predicate_parser_data(), combined, &patterns)
.expect("combined grammar maps only the parser-rule predicate");
assert_eq!(templates.len(), 1, "only the parser-rule predicate maps");
assert_eq!(templates[0].0, (0, 0), "mapped to rule `s` pred 0");
assert_eq!(templates[0].1, PredicateTemplate::Hook);
}
#[test]
fn parser_predicate_scan_keeps_predicate_when_header_unresolvable() {
// A `;` inside a comment above the rule defeats the brace-counting
// header scraper (`statement_rule_header` returns None). The predicate
// filter must KEEP such a block rather than drop a real parser
// predicate — regression for SemPredEvalParser/ValidateInDFA, whose
// comment `// ';' helps ...` made rule `a`'s predicates vanish.
let grammar = concat!(
"grammar T;\n",
"s : a ';' a;\n",
"// ';' helps us resynchronize without consuming\n",
"a : {<False()>}? ID\n",
" | {<True()>}? INT\n",
" ;\n",
"ID : 'a'..'z'+ ;\n",
);
// rule_names carries only parser rules `s`, `a` (as a parser .interp would).
let rule_names = ["s".to_owned(), "a".to_owned()];
let mut offset = 0;
let mut kept = Vec::new();
while let Some(block) = next_predicate_action_block(grammar, offset) {
offset = block.after_brace;
if predicate_block_included(grammar, block.open_brace, &rule_names) {
kept.push(block.body.to_owned());
}
}
assert_eq!(
kept,
["<False()>".to_owned(), "<True()>".to_owned()],
"both rule-`a` predicates must survive the comment-with-semicolon header"
);
}
#[test]
fn parses_predicate_fail_option_message() {
let grammar = "a : a ID {<False()>}?<fail='custom message'> | ID ;";
let block =
next_predicate_action_block(grammar, 0).expect("predicate block should be present");
assert_eq!(
predicate_fail_message(grammar, block.after_brace),
Some("custom message".to_owned())
);
assert_eq!(
predicate_template_with_fail_message(
PredicateTemplate::False,
"custom message".to_owned(),
),
PredicateTemplate::FalseWithMessage {
message: "custom message".to_owned()
}
);
// A non-constant-false predicate (hook, member, lookahead, …) preserves
// its `<fail=...>` message via the transparent `WithFailMessage` wrapper
// rather than discarding it.
let wrapped =
predicate_template_with_fail_message(PredicateTemplate::Hook, "hook failed".to_owned());
assert_eq!(
wrapped,
PredicateTemplate::WithFailMessage {
inner: Box::new(PredicateTemplate::Hook),
message: "hook failed".to_owned(),
}
);
// The wrapper is transparent to evaluation and generatability, and it
// exposes the message.
assert_eq!(
predicate_effective_template(&wrapped),
&PredicateTemplate::Hook
);
assert!(can_generate_parser_predicate(&wrapped));
assert_eq!(
predicate_template_fail_message(&wrapped),
Some("hook failed")
);
// Disposition follows the inner: a wrapped hook is still `Hooked`.
assert_eq!(
predicate_template_disposition(Some(&wrapped), SemUnknownPolicy::AssumeTrue),
SemanticsDisposition::Hooked
);
// A later `<fail=...>` replaces the message rather than nesting wrappers.
let rewrapped = predicate_template_with_fail_message(wrapped, "again".to_owned());
assert_eq!(
rewrapped,
PredicateTemplate::WithFailMessage {
inner: Box::new(PredicateTemplate::Hook),
message: "again".to_owned(),
}
);
}
#[test]
fn parser_action_source_slots_ignore_signature_templates() {
let rule_names = vec!["root".to_owned(), "continue".to_owned()];
let grammar = r#"root : {<write("$text")>} continue ;
continue returns [<IntArg("return")>] : {<AssignLocal("$return","0")>} ;"#;
let spans = parser_action_source_block_slots(grammar, &rule_names);
assert_eq!(spans.len(), 2, "only authored action blocks produce slots");
assert_eq!(spans[0].body, r#"<write("$text")>"#);
assert_eq!(spans[0].rule_index, 0);
assert_eq!(spans[1].body, r#"<AssignLocal("$return","0")>"#);
assert_eq!(spans[1].rule_index, 1);
}
#[test]
fn parser_action_source_slots_ignore_rule_exception_handlers() {
let rule_names = vec!["s".to_owned()];
let grammar = r#"parser grammar T;
s : ID { native(); } EOF ;
catch [antlr4_runtime::AntlrError error] { recover(error); }
finally { cleanup(); }
ID : [a-z]+ ;
"#;
let spans = parser_action_source_block_slots(grammar, &rule_names);
assert_eq!(spans.len(), 1, "only rule-body actions produce slots");
assert_eq!(spans[0].body.trim(), "native();");
assert_eq!(spans[0].rule_index, 0);
}
#[test]
fn parser_action_source_slots_keep_rule_refs_ending_in_metadata_words() {
let rule_names = vec!["s".to_owned()];
let grammar = r#"parser grammar T;
s : mytokens { first(); } mychannels { second(); } ;
"#;
let spans = parser_action_source_block_slots(grammar, &rule_names);
assert_eq!(
spans
.iter()
.map(|slot| slot.body.trim())
.collect::<Vec<_>>(),
["first();", "second();"]
);
}
#[test]
fn parser_action_assignment_is_per_rule_and_left_recursion_aware() {
let data = multi_rule_action_parser_data();
let grammar = r#"parser grammar T;
s : {one();} A ;
expr : expr PLUS {op();} expr
| {primary();} A
;
"#;
let assigned = assign_states_to_parser_action_slots(
&data,
grammar,
parser_action_source_block_slots(grammar, &data.rule_names),
)
.expect("action slots should assign");
let by_state = assigned
.iter()
.map(|(state, slot)| (*state, (slot.rule_index, slot.body.as_str())))
.collect::<BTreeMap<_, _>>();
assert_eq!(by_state.get(&0), Some(&(0, "one();")));
assert_eq!(by_state.get(&5), Some(&(1, "primary();")));
assert_eq!(by_state.get(&6), Some(&(1, "op();")));
assert!(
!by_state.contains_key(&4),
"synthetic state 4 has no source body"
);
}
#[test]
fn parses_supported_predicate_helpers() {
assert_eq!(
parse_invoke_predicate(r#"True():Invoke_pred()"#),
Some(PredicateTemplate::Invoke { value: true })
);
assert_eq!(
parse_invoke_predicate(r#"False():Invoke_pred()"#),
Some(PredicateTemplate::Invoke { value: false })
);
assert_eq!(
parse_predicate_template(r#"ParserPropertyCall({$parser}, "Property()")"#),
Some(PredicateTemplate::True)
);
assert_eq!(
parse_predicate_template("true"),
Some(PredicateTemplate::True)
);
assert_eq!(
parse_predicate_template("0==0"),
Some(PredicateTemplate::True)
);
assert_eq!(
parse_predicate_template("0 != 0"),
Some(PredicateTemplate::False)
);
assert_eq!(
parse_val_equals_predicate(r#"ValEquals("$i","2")"#),
Some(PredicateTemplate::LocalIntEquals { value: 2 })
);
assert_eq!(
parse_raw_local_int_less_or_equal_predicate("5 >= $_p"),
Some(PredicateTemplate::LocalIntLessOrEqual { value: 5 })
);
assert_eq!(
parse_predicate_template("this.IsRightArrow()"),
Some(PredicateTemplate::TokenPairAdjacent)
);
assert_eq!(
parse_predicate_template("this.IsLocalVariableDeclaration()"),
Some(PredicateTemplate::ContextChildRuleTextNotEquals {
rule_name: "local_variable_type".to_owned(),
text: "var".to_owned(),
})
);
}
#[test]
fn maps_kotlin_rcurl_java_action_to_lexer_pop_mode() {
for body in [
"popMode()",
"popMode();",
"this.popMode()",
"this.popMode();",
"if (!_modeStack.isEmpty()) { popMode(); }",
"if (!this._modeStack.isEmpty()) { popMode(); }",
"if (!_modeStack.isEmpty()) popMode()",
"if (!this._modeStack.isEmpty()) popMode();",
] {
assert_eq!(
parse_lexer_pop_mode_action(body),
Some(ActionTemplate::LexerPopMode),
"{body}"
);
}
let grammar = r#"
lexer grammar KotlinLexer;
LCURL: '{' -> pushMode(DEFAULT_MODE);
RCURL: '}' { if (!_modeStack.isEmpty()) { popMode(); } };
LineStrRef: '${' -> pushMode(DEFAULT_MODE);
"#;
let rule_names = vec![
"LCURL".to_owned(),
"RCURL".to_owned(),
"LineStrRef".to_owned(),
];
let actions = extract_lexer_action_templates(grammar, &rule_names);
assert_eq!(actions, [ActionTemplate::LexerPopMode]);
let method = render_lexer_action_method(&[((1, 0), actions[0].clone())]);
assert!(method.contains("fn run_action"));
assert!(method.contains("_base.pop_mode();"));
}
#[test]
fn lexer_action_scan_ignores_braces_inside_character_sets() {
let grammar = r#"
lexer grammar L;
LETTER: [\p{L}{}]+;
ESCAPED_RBRACK: [\]]+;
RCURL: '}' { popMode(); };
"#;
let rule_names = vec![
"LETTER".to_owned(),
"ESCAPED_RBRACK".to_owned(),
"RCURL".to_owned(),
];
let actions = extract_lexer_action_templates(grammar, &rule_names);
assert_eq!(actions, [ActionTemplate::LexerPopMode]);
}
#[test]
fn lexer_action_scan_resolves_rule_after_block_comment() {
// A `/* ... */` block comment between the previous rule's `;` and a rule
// that carries a custom action (the shape of Kotlin's `RCURL`) must not
// hide the rule name. Before the fix the action was mis-attributed and
// dropped, so the grammar yielded zero action templates while the lexer
// ATN had one — a hard count-mismatch abort.
let grammar = "lexer grammar L;\n\
LCURL: '{' -> pushMode(DEFAULT_MODE);\n\
/*\n\
* doc comment sitting above the action rule\n\
*/\n\
RCURL: '}' { if (!_modeStack.isEmpty()) { popMode(); } };\n\
ID: [a-z]+ ;\n";
let rule_names = vec!["LCURL".to_owned(), "RCURL".to_owned(), "ID".to_owned()];
let actions = extract_lexer_action_templates(grammar, &rule_names);
// Both the pushMode command and the guarded-popMode idiom resolve; the
// action is attributed to `RCURL`, not skipped.
assert_eq!(actions, [ActionTemplate::LexerPopMode]);
}
#[test]
fn rule_header_identifier_skips_comments() {
// The rule-name reader skips leading `//` and `/* */` comments so the
// rule name that follows resolves (the header for an action after a
// doc-commented rule).
assert_eq!(
rule_header_start_identifier("/* one-line */ RCURL"),
Some("RCURL")
);
assert_eq!(
rule_header_start_identifier("/*\n * multi\n * line\n */\nRCURL"),
Some("RCURL")
);
assert_eq!(
rule_header_start_identifier("// line\n/* block */\nname_x"),
Some("name_x")
);
// A rule-level `@init {...}` clause between the name and `:` is skipped,
// so the name still resolves.
assert_eq!(
rule_header_start_identifier("s @init { init(); }"),
Some("s")
);
// The statement-start reader takes the FIRST identifier, so a rule with a
// `returns [...]` / `locals [...]` clause resolves to the rule name, not
// the option keyword or a `[...]` type. (Regression: taking the last
// identifier resolved `continue returns [int x]` to `returns`/`x`.)
assert_eq!(
rule_header_start_identifier("continue returns [int x]"),
Some("continue")
);
assert_eq!(
rule_header_start_identifier("s @init { init(); }"),
Some("s")
);
assert_eq!(
rule_header_start_identifier("/* doc */ RCURL"),
Some("RCURL")
);
// `fragment` lexer rules keep their name.
assert_eq!(
rule_header_start_identifier("fragment DIGIT"),
Some("DIGIT")
);
// A grammar-level `tokens { ... }` / `options { ... }` declaration
// precedes the first rule with no terminating `;`; it must be skipped so
// the following rule name resolves (composite/imported grammars).
assert_eq!(
rule_header_start_identifier("tokens { A, B, C }\nx"),
Some("x")
);
assert_eq!(
rule_header_start_identifier("options { k = v; }\nexpr"),
Some("expr")
);
}
#[test]
fn parser_action_source_attribution_across_imported_grammar_with_tokens_block() {
// Combined/imported grammar source (delegator + slave concatenated, as
// the runtime-testsuite builds it): the slave rule `x`'s translatable
// action must attribute to `x` — a `tokens { ... }` block before it (no
// `;`) previously derailed the rule-name scan and dropped the source span,
// so `semantics.json` could not report the authored unsupported action.
let grammar = "grammar M;\n\
import S;\n\
s : x INT;\n\
parser grammar S;\n\
tokens { A, B, C }\n\
x : 'x' INT {<writeln(\"S.x\")>};\n\
INT : '0'..'9'+ ;\n";
let rule_names = vec!["s".to_owned(), "x".to_owned()];
let spans = parser_action_source_block_slots(grammar, &rule_names);
assert_eq!(
spans
.iter()
.map(|span| span.body.as_str())
.collect::<Vec<_>>(),
[r#"<writeln("S.x")>"#],
"the imported rule's action source span must not be dropped"
);
assert_eq!(spans[0].rule_index, 1);
}
#[test]
fn lexer_action_scan_keeps_unsupported_actions_after_prior_st_markup() {
let grammar = r#"
lexer grammar L;
I : ({<PlusText("stuff fail: "):writeln()>} 'a'
| {<PlusText("stuff0:"):writeln()>}
'a' {<PlusText("stuff1: "):writeln()>}
'b' {<PlusText("stuff2: "):writeln()>})
{<Text():writeln()>} ;
WS : (' '|'\n') -> skip ;
J : .;
"#;
let rule_names = vec!["I".to_owned(), "WS".to_owned(), "J".to_owned()];
let actions = extract_lexer_action_templates(grammar, &rule_names);
assert_eq!(actions.len(), 5);
assert!(actions.iter().all(|action| matches!(
action,
ActionTemplate::UnsupportedLexerAction { rule_name, .. } if rule_name == "I"
)));
}
#[test]
fn unsupported_lexer_action_renders_todo_marker() {
let grammar = r#"
lexer grammar L;
ID: [a-z]+ { customJava(); };
"#;
let rule_names = vec!["ID".to_owned()];
let actions = extract_lexer_action_templates(grammar, &rule_names);
assert_eq!(
actions,
[ActionTemplate::UnsupportedLexerAction {
rule_name: "ID".to_owned(),
body: "customJava();".to_owned(),
}]
);
assert_eq!(
render_lexer_action_statement(&actions[0]),
"/* TODO unsupported embedded lexer action in rule ID: {customJava();}; rewrite target-specific actions as portable lexer commands where possible */"
);
let method = render_lexer_action_method(&[((1, 0), actions[0].clone())]);
assert!(method.contains("TODO unsupported embedded lexer action in rule ID"));
assert!(!method.contains("fn run_action"));
assert_eq!(rust_block_comment_text("a */ b"), "a * / b");
let error = reject_unsupported_lexer_action_templates(&actions, false).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert!(error.to_string().contains(
"unsupported embedded lexer action in rule ID: {customJava();}; \
rewrite target-specific actions as portable lexer commands where possible"
));
reject_unsupported_lexer_action_templates(&actions, true)
.expect("unsupported-only lexer actions should be allowed in compatibility mode");
}
#[test]
fn mixed_supported_and_unsupported_lexer_actions_fail_even_when_allowed() {
let actions = vec![
ActionTemplate::UnsupportedLexerAction {
rule_name: "ID".to_owned(),
body: "setType(Foo);".to_owned(),
},
ActionTemplate::LexerPopMode,
];
let error = reject_unsupported_lexer_action_templates(&actions, true).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert!(error.to_string().contains(
"unsupported embedded lexer action in rule ID: {setType(Foo);}; \
rewrite target-specific actions as portable lexer commands where possible"
));
}
#[test]
fn zero_lexer_action_templates_defer_every_coordinate_to_policy() {
let data = custom_action_lexer_data();
let parser_only_grammar = "parser grammar P;\ns : A ;\n";
let actions = lexer_action_templates(
&data,
parser_only_grammar,
false,
&SemPatternFile::default(),
)
.expect("zero matched templates should defer instead of aborting generation");
assert!(actions.is_empty());
let entries = collect_lexer_semantics(
&data,
Some(parser_only_grammar),
false,
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("deferred actions should remain inventoried");
assert_eq!(entries.len(), 2);
assert_eq!(
entries
.iter()
.map(|entry| (
entry.rule_name.as_deref(),
entry.rule_index,
entry.index,
entry.disposition,
))
.collect::<Vec<_>>(),
[
(Some("A"), Some(0), Some(0), SemanticsDisposition::Ignored,),
(Some("B"), Some(1), Some(1), SemanticsDisposition::Ignored,),
]
);
let manifest = render_semantics_manifest(
SemUnknownPolicy::AssumeTrue,
&[],
&[("lexer", "L".to_owned(), entries)],
);
assert_eq!(manifest.matches("\"kind\": \"lexer-action\"").count(), 2);
let hooked_entries = collect_lexer_semantics(
&data,
Some(parser_only_grammar),
false,
SemUnknownPolicy::Hook,
&SemPatternFile::default(),
)
.expect("hook policy should collect deferred actions");
assert!(hooked_entries.iter().all(|entry| {
entry.disposition == SemanticsDisposition::Hooked && entry.template.is_none()
}));
let hooked_module = render_lexer(
"L",
&data,
Some(parser_only_grammar),
false,
SemUnknownPolicy::Hook,
&SemPatternFile::default(),
false,
)
.expect("hook policy should generate a lexer");
assert!(hooked_module.contains("next_token_compiled_with_semantic_dispatch"));
let strict_entries = collect_lexer_semantics(
&data,
Some(parser_only_grammar),
false,
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("strict policy should inventory before failing");
let error = enforce_sem_unknown(SemUnknownPolicy::Error, &strict_entries)
.expect_err("strict policy should reject both deferred actions");
assert!(
error
.to_string()
.contains("2 semantic coordinate(s) have no Rust implementation")
);
}
#[test]
fn partial_lexer_action_template_matches_still_fail() {
let grammar = "lexer grammar L;\nA : 'a' { popMode(); };\n";
let error = lexer_action_templates(
&custom_action_lexer_data(),
grammar,
true,
&SemPatternFile::default(),
)
.expect_err("one template cannot be safely aligned with two ATN actions");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert!(error.to_string().contains(
"lexer ATN has 2 custom action(s), but grammar source yielded 1 lexer action template(s)"
));
}
#[test]
fn lexer_action_diagnostic_summary_truncates_on_char_boundary() {
let body = format!("{}\u{00e9} tail", "a".repeat(95));
let summary = one_line_action_body(&body);
assert_eq!(summary, format!("{}...", "a".repeat(95)));
}
fn linear_rule_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
3
);
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(
1,
ParserTransitionSpec::Atom {
target: 2,
label: 1,
},
)
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 3,
label: TOKEN_EOF,
},
)
.expect("transition");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![3])
.expect("rule stop states");
finish_atn(atn)
}
fn block_decision_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::BlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
5
);
atn.set_end_state(1, 4).expect("block end state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 4,
label: 1,
},
)
.expect("transition");
atn.add_transition(
3,
ParserTransitionSpec::Atom {
target: 4,
label: 2,
},
)
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_decision_state(1).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![5])
.expect("rule stop states");
finish_atn(atn)
}
fn star_loop_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopEntry, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopBack, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
5
);
atn.set_loop_back_state(3, 4).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 4,
label: 1,
},
)
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_decision_state(1).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![5])
.expect("rule stop states");
finish_atn(atn)
}
fn plus_loop_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::PlusBlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::PlusLoopBack, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
6
);
atn.set_end_state(1, 3).expect("block end state");
atn.set_loop_back_state(5, 4).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 3,
label: 1,
},
)
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 })
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 })
.expect("transition");
atn.add_decision_state(4).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![6])
.expect("rule stop states");
finish_atn(atn)
}
fn plus_block_decision_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::PlusBlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::PlusLoopBack, Some(0))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
6
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
7
);
atn.set_end_state(1, 4).expect("block end state");
atn.set_loop_back_state(6, 5).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(
2,
ParserTransitionSpec::Atom {
target: 4,
label: 1,
},
)
.expect("transition");
atn.add_transition(
3,
ParserTransitionSpec::Atom {
target: 4,
label: 2,
},
)
.expect("transition");
atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 })
.expect("transition");
atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 })
.expect("transition");
atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 })
.expect("transition");
atn.add_decision_state(1).expect("decision state");
atn.add_decision_state(5).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![7])
.expect("rule stop states");
finish_atn(atn)
}
fn left_recursive_rule_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopEntry, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::StarBlockStart, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
6
);
assert_eq!(
atn.add_state(AtnStateKind::LoopEnd, Some(0))
.expect("state")
.index(),
7
);
assert_eq!(
atn.add_state(AtnStateKind::StarLoopBack, Some(0))
.expect("state")
.index(),
8
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
9
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
10
);
atn.set_left_recursive_rule(0)
.expect("left-recursive rule start");
atn.set_precedence_rule_decision(2)
.expect("precedence decision");
atn.set_end_state(3, 6).expect("block end state");
atn.set_loop_back_state(7, 8).expect("loop back state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(
1,
ParserTransitionSpec::Atom {
target: 2,
label: 1,
},
)
.expect("transition");
atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 })
.expect("transition");
atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 7 })
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 })
.expect("transition");
atn.add_transition(
4,
ParserTransitionSpec::Precedence {
target: 5,
precedence: 2,
},
)
.expect("transition");
atn.add_transition(
5,
ParserTransitionSpec::Atom {
target: 10,
label: 2,
},
)
.expect("transition");
atn.add_transition(
10,
ParserTransitionSpec::Rule {
target: 0,
rule_index: 0,
follow_state: 6,
precedence: 3,
},
)
.expect("transition");
atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 8 })
.expect("transition");
atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 2 })
.expect("transition");
atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 9 })
.expect("transition");
atn.add_decision_state(2).expect("decision state");
atn.add_decision_state(3).expect("decision state");
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![9])
.expect("rule stop states");
finish_atn(atn)
}
fn entry_candidate_atn() -> ParserAtn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(1))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(1))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(2))
.expect("state")
.index(),
5
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(2))
.expect("state")
.index(),
6
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(2))
.expect("state")
.index(),
7
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(3))
.expect("state")
.index(),
8
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(3))
.expect("state")
.index(),
9
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(3))
.expect("state")
.index(),
10
);
atn.add_transition(
0,
ParserTransitionSpec::Rule {
target: 3,
rule_index: 1,
follow_state: 2,
precedence: 0,
},
)
.expect("transition");
atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 })
.expect("transition");
atn.add_transition(
5,
ParserTransitionSpec::Rule {
target: 3,
rule_index: 1,
follow_state: 7,
precedence: 0,
},
)
.expect("transition");
atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 6 })
.expect("transition");
atn.add_transition(
8,
ParserTransitionSpec::Rule {
target: 8,
rule_index: 3,
follow_state: 10,
precedence: 0,
},
)
.expect("transition");
atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 9 })
.expect("transition");
atn.set_rule_to_start_state(vec![0, 3, 5, 8])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![1, 4, 6, 9])
.expect("rule stop states");
finish_atn(atn)
}
fn minimal_parser_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned())],
rule_names: vec!["s".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 1, 1, // version, parser grammar, max token type
2, // states
2, 0, // rule start
7, 0, // rule stop
0, // non-greedy states
0, // precedence states
1, // rules
0, // rule 0 start
0, // modes
0, // sets
1, // transitions
0, 1, 1, 0, 0, 0, // epsilon
0, // decisions
],
}
}
/// Parser `.interp` fixture whose ATN carries one parser action transition:
/// `s : { ... } A ;`.
fn action_parser_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned())],
rule_names: vec!["s".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 1, 1, // version, parser grammar, max token type
4, // states
2, 0, // state 0: rule start
1, 0, // state 1: basic
1, 0, // state 2: basic
7, 0, // state 3: rule stop
0, // non-greedy states
0, // precedence states
1, // rules
0, // rule 0 start
0, // modes
0, // sets
3, // transitions
0, 1, 6, 0, 0, 0, // action rule 0 action 0
1, 2, 5, 1, 0, 0, // atom A
2, 3, 1, 0, 0, 0, // epsilon
0, // decisions
],
}
}
/// Parser `.interp` fixture with an authored action in rule `s`, then one
/// synthetic and two authored action states in left-recursive rule `expr`.
fn multi_rule_action_parser_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned()), Some("'+'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned()), Some("PLUS".to_owned())],
rule_names: vec!["s".to_owned(), "expr".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 1, 2, // version, parser grammar, max token type
9, // states
2, 0, // state 0: rule s start, authored action source
1, 0, // state 1: basic
1, 0, // state 2: basic
7, 0, // state 3: rule s stop
2, 1, // state 4: rule expr start, synthetic action source
1, 1, // state 5: primary action source
1, 1, // state 6: operator action source
1, 1, // state 7: basic
7, 1, // state 8: rule expr stop
0, // non-greedy states
0, // precedence states
2, // rules
0, // rule 0 start
4, // rule 1 start
0, // modes
0, // sets
7, // transitions
0, 1, 6, 0, 0, 0, // action rule s
1, 2, 5, 1, 0, 0, // atom A
2, 3, 1, 0, 0, 0, // epsilon
4, 5, 6, 1, 0, 0, // synthetic action in expr
5, 6, 6, 1, 1, 0, // authored primary action in expr
6, 7, 6, 1, 2, 0, // authored operator action in expr
7, 8, 5, 1, 0, 0, // atom A
0, // decisions
],
}
}
/// Parser `.interp` fixture whose ATN carries one semantic-predicate
/// transition at coordinate `(rule 0, pred 0)`: `s : {…}? A ;`.
fn predicate_parser_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned())],
rule_names: vec!["s".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 1, 1, // version, parser grammar, max token type
4, // states
2, 0, // state 0: rule start
1, 0, // state 1: basic
1, 0, // state 2: basic
7, 0, // state 3: rule stop
0, // non-greedy states
0, // precedence states
1, // rules
0, // rule 0 start
0, // modes
0, // sets
3, // transitions
0, 1, 4, 0, 0, 0, // predicate rule 0 pred 0
1, 2, 5, 1, 0, 0, // atom A
2, 3, 1, 0, 0, 0, // epsilon
0, // decisions
],
}
}
/// Lexer `.interp` fixture with one semantic-predicate transition at
/// coordinate `(rule 0, pred 0)`.
fn predicate_lexer_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned())],
rule_names: vec!["A".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 0, 1, // version, lexer grammar, max token type
5, // states
6, -1, // state 0: mode token start
2, 0, // state 1: rule start
1, 0, // state 2: predicate source
1, 0, // state 3: token source
7, 0, // state 4: rule stop
0, // non-greedy states
0, // precedence states
1, // rules
1, 1, // rule 0 start and token type
1, // modes
0, // mode 0 start
0, // sets
4, // transitions
0, 1, 1, 0, 0, 0, // epsilon to rule start
1, 2, 4, 0, 0, 0, // predicate rule 0 pred 0
2, 3, 5, 1, 0, 0, // atom A
3, 4, 1, 0, 0, 0, // epsilon to stop
1, // decisions
0, // mode start decision
0, // lexer actions
],
}
}
/// Lexer `.interp` fixture for:
/// `A : 'a' { customJava(); }; B : 'b' { customJava(); };`.
#[rustfmt::skip]
fn custom_action_lexer_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned()), Some("'b'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned()), Some("B".to_owned())],
rule_names: vec!["A".to_owned(), "B".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned(), "HIDDEN".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 0, 2, 11, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1,
1, 1, 0, 0, 2, 1, 1, 3, 2, 1, 0, 0, 10, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0,
0, 1, 5, 1, 0, 0, 0, 3, 8, 1, 0, 0, 0, 5, 6, 5, 97, 0, 0, 6, 7, 6, 0, 0,
0, 7, 2, 1, 0, 0, 0, 8, 9, 5, 98, 0, 0, 9, 10, 6, 1, 1, 0, 10, 4, 1, 0, 0,
0, 1, 0, 2, 1, 0, 0, 1, 1, 1,
],
}
}
/// Parser `.interp` fixture for
/// `s locals [boolean seen=false] : {$seen=true;} {$seen}? A ;`.
fn portable_bool_parser_data() -> InterpData {
InterpData {
literal_names: vec![None, Some("'a'".to_owned())],
symbolic_names: vec![None, Some("A".to_owned())],
rule_names: vec!["s".to_owned()],
channel_names: vec!["DEFAULT_TOKEN_CHANNEL".to_owned()],
mode_names: vec!["DEFAULT_MODE".to_owned()],
atn: vec![
4, 1, 1, // version, parser grammar, max token type
4, // states
2, 0, // state 0: rule start/action source
1, 0, // state 1: predicate source
1, 0, // state 2: token source
7, 0, // state 3: rule stop
0, // non-greedy states
0, // precedence states
1, // rules
0, // rule 0 start
0, // modes
0, // sets
3, // transitions
0, 1, 6, 0, 0, 0, // action rule 0 action 0
1, 2, 4, 0, 0, 0, // predicate rule 0 pred 0
2, 3, 5, 1, 0, 0, // atom A
0, // decisions
],
}
}
const PORTABLE_BOOL_GRAMMAR: &str = "parser grammar S;\n\
s locals [boolean seen=false]\n\
: { $seen = true; } {$seen}?<fail='not seen'> A\n\
;\n";
const PREDICATE_GRAMMAR: &str = "parser grammar S;\ns : {isTypeName()}? A ;\n";
#[test]
fn sem_unknown_flag_values_parse() {
assert_eq!(
SemUnknownPolicy::parse_flag("error").expect("error parses"),
SemUnknownPolicy::Error
);
assert_eq!(
SemUnknownPolicy::parse_flag("assume-true").expect("assume-true parses"),
SemUnknownPolicy::AssumeTrue
);
assert_eq!(
SemUnknownPolicy::parse_flag("assume-false").expect("assume-false parses"),
SemUnknownPolicy::AssumeFalse
);
assert_eq!(
SemUnknownPolicy::parse_flag("hook").expect("hook parses"),
SemUnknownPolicy::Hook
);
assert!(SemUnknownPolicy::parse_flag("bogus").is_err());
}
#[test]
fn set_lexer_predicate_template_replaces_or_appends() {
// A per-coordinate override must WIN over a built-in translation, so
// setting a covered coordinate replaces its template rather than adding
// a duplicate arm; an uncovered coordinate is appended.
let mut predicates = vec![((0, 0), PredicateTemplate::True)];
set_lexer_predicate_template(&mut predicates, (0, 0), PredicateTemplate::False);
assert_eq!(
predicates,
[((0, 0), PredicateTemplate::False)],
"replaces covered"
);
set_lexer_predicate_template(&mut predicates, (1, 2), PredicateTemplate::True);
assert_eq!(
predicates,
[
((0, 0), PredicateTemplate::False),
((1, 2), PredicateTemplate::True)
],
"appends uncovered"
);
}
#[test]
fn parser_action_assume_override_gets_noop_disposition() {
// Assumed parser actions get explicit no-op arms; hook/error overrides
// keep falling through to the parser action hook.
let data = predicate_parser_data(); // rule 0 = "s"
let mut action_state_rules = BTreeMap::new();
action_state_rules.insert(4_usize, 0_usize); // action state 4 belongs to rule `s`
for dispose in ["assume-true", "assume-false"] {
let patterns = parse_sem_patterns(&format!(
"version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\ndispose = \"{dispose}\"\n"
))
.expect("pattern file parses");
assert!(
parser_action_assume_overridden(&patterns, &data, &action_state_rules, 4),
"dispose {dispose}: an action state in rule `s` is assumed"
);
}
let hook_patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
assert!(
!parser_action_assume_overridden(&hook_patterns, &data, &action_state_rules, 4),
"hook overrides should keep routing through the hook arm"
);
assert!(
!parser_action_assume_overridden(
&SemPatternFile::default(),
&data,
&action_state_rules,
4
),
"no override -> concrete arm is kept"
);
}
#[test]
fn sem_pattern_file_lowers_exact_predicate_body() {
let patterns = parse_sem_patterns(
r#"
[[pattern]]
match = "isTypeName()"
lower = "bool(false)"
"#,
)
.expect("pattern file parses");
let predicates =
parser_predicate_templates(&predicate_parser_data(), PREDICATE_GRAMMAR, &patterns)
.expect("pattern should lower predicate");
assert_eq!(predicates[0].1, PredicateTemplate::False);
}
#[test]
fn strip_toml_comment_respects_quoted_strings() {
// A `#` outside quotes starts a comment.
assert_eq!(strip_toml_comment("key = 1 # trailing"), "key = 1 ");
assert_eq!(strip_toml_comment("# whole line"), "");
// A `#` inside a basic or literal string is NOT a comment.
assert_eq!(
strip_toml_comment(r#"match = "text == '#'""#),
r#"match = "text == '#'""#
);
assert_eq!(
strip_toml_comment(r##"lower = 'str("#")'"##),
r##"lower = 'str("#")'"##
);
// A `#` after a closed string is still a comment.
assert_eq!(
strip_toml_comment(r#"match = "a" # note"#),
r#"match = "a" "#
);
// A `\"` escape inside a basic string does not close it, so a later `#`
// stays inside the string.
assert_eq!(
strip_toml_comment(r##"match = "a\"#b""##),
r##"match = "a\"#b""##
);
}
#[test]
fn sem_patterns_keep_hash_inside_quoted_match() {
// A `#` inside the quoted `match` body must survive parsing, not be
// truncated as a comment (which would silently change the pattern).
let patterns = parse_sem_patterns(
"[[pattern]]\nmatch = \"col == '#'\" # a real comment\nlower = \"bool(false)\"\n",
)
.expect("pattern file parses");
let template = patterns
.predicate_template(SemanticsKind::ParserPredicate, "col == '#'")
.expect("pattern lookup should not fail")
.expect("the '#'-bearing match body must be retained");
assert_eq!(template, PredicateTemplate::False);
}
#[test]
fn parse_toml_scalar_strips_single_quoted_literals() {
// TOML literal strings are single-quoted with no escape processing.
// `strip_toml_comment` already treats `'...'` as a string, so the scalar
// parser must strip those quotes too (verbatim body).
assert_eq!(parse_toml_scalar("'assume-false'"), "assume-false");
assert_eq!(parse_toml_scalar(" 'hook' "), "hook");
// A literal string is verbatim: a backslash is not an escape.
assert_eq!(parse_toml_scalar(r"'a\nb'"), r"a\nb");
// Double-quoted basic strings still unescape.
assert_eq!(parse_toml_scalar(r#""a\nb""#), "a\nb");
// A bare scalar and a lone quote are unchanged.
assert_eq!(parse_toml_scalar("42"), "42");
assert_eq!(parse_toml_scalar("'"), "'");
}
#[test]
fn sem_patterns_accept_single_quoted_dispose() {
// A single-quoted (TOML literal) `dispose` must resolve to the disposition,
// not keep its quotes (which would fail to match / be rejected).
let patterns = parse_sem_patterns(
"[[coordinate]]\nkind = 'predicate'\nrule = 's'\nindex = 0\ndispose = 'assume-false'\n",
)
.expect("pattern file parses");
assert_eq!(
patterns.coordinate_predicate_template(
SemanticsKind::ParserPredicate,
Some("s"),
Some(0),
),
Some(Some(PredicateTemplate::False)),
"single-quoted dispose must resolve to assume-false"
);
}
#[test]
fn semantic_helper_calls_capture_kind_negation_and_literals() {
assert_eq!(
parse_semantic_helper_call(
r#"!this.matches("marker", true, -2)"#,
SemanticsKind::LexerPredicate,
),
Some(SemanticHelperCall {
name: "matches".to_owned(),
arguments: vec![
SemanticLiteral::String("marker".to_owned()),
SemanticLiteral::Bool(true),
SemanticLiteral::Integer(-2),
],
negated: true,
})
);
assert_eq!(
parse_semantic_helper_call("this.HandleAction();", SemanticsKind::LexerAction,),
Some(SemanticHelperCall {
name: "HandleAction".to_owned(),
arguments: Vec::new(),
negated: false,
})
);
assert!(
parse_semantic_helper_call("this.n(dynamicValue)", SemanticsKind::ParserPredicate,)
.is_none()
);
assert_eq!(
parse_semantic_helper_call(
r"this.n('line\n\'quoted\'')",
SemanticsKind::ParserPredicate,
)
.expect("single-quoted literal parses")
.arguments,
[SemanticLiteral::String("line\n'quoted'".to_owned())]
);
}
#[test]
fn semantic_helper_patterns_are_scoped_by_kind_and_signature() {
let patterns = parse_sem_patterns(
"version = 1\n[[helper]]\nkind = \"lexer-action\"\nname = \"handle\"\nreturns = \"unit\"\nlower = \"hook\"\n[[helper]]\nname = \"n\"\narguments = \"string\"\nreturns = \"bool\"\nlower = \"hook\"\n",
)
.expect("pattern file parses");
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerAction, "this.handle();")
.expect("matching cannot fail")
.is_some()
);
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerPredicate, "this.handle()")
.expect("matching cannot fail")
.is_none()
);
let call = patterns
.hook_helper_call(SemanticsKind::ParserPredicate, r#"this.n("value")"#)
.expect("matching cannot fail")
.expect("string argument matches");
assert_eq!(
call.arguments,
[SemanticLiteral::String("value".to_owned())]
);
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerPredicate, r#"this.n("value")"#)
.expect("matching cannot fail")
.is_some()
);
assert!(
patterns
.hook_helper_call(SemanticsKind::LexerAction, r#"this.n("value");"#)
.expect("matching cannot fail")
.is_none()
);
}
#[test]
fn lexer_typed_hook_signatures_reject_normalized_conflicts() {
let mappings = [
LexerTypedHookMapping {
rule_index: 0,
coordinate_index: 0,
kind: LexerTypedHookKind::Action,
method_name: "handle_value".to_owned(),
call: SemanticHelperCall {
name: "HandleValue".to_owned(),
arguments: vec![SemanticLiteral::String("x".to_owned())],
negated: false,
},
},
LexerTypedHookMapping {
rule_index: 1,
coordinate_index: 0,
kind: LexerTypedHookKind::Action,
method_name: "handle_value".to_owned(),
call: SemanticHelperCall {
name: "handle_value".to_owned(),
arguments: vec![SemanticLiteral::Integer(1)],
negated: false,
},
},
];
let error = validate_lexer_typed_hook_signatures(&mappings)
.expect_err("conflicting normalized signatures must fail generation");
assert!(error.to_string().contains("conflicting literal signatures"));
}
#[test]
fn coordinate_override_routes_parser_predicate_to_typed_hook() {
let patterns = parse_sem_patterns(
r#"
[[coordinate]]
kind = "predicate"
rule = "s"
index = 0
dispose = "hook"
"#,
)
.expect("pattern file parses");
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::AssumeTrue,
&patterns,
)
.expect("collection should succeed");
assert_eq!(entries[0].disposition, SemanticsDisposition::Hooked);
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
patterns: Some(&patterns),
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(module.contains("pub trait SParserHooks"));
assert!(module.contains("fn is_type_name"));
assert!(module.contains("(0, 0) => Some(self.0.is_type_name(ctx))"));
assert!(module.contains("PExpr::Hook"));
// The typed action escape hatch must report handled actions: the trait's
// `custom_action` returns `bool` and the adapter propagates it (so a
// typed action hook satisfies a hook/error policy instead of being
// treated as unhandled and failing loud).
assert!(
module.contains("_action: antlr4_runtime::ParserAction) -> bool"),
"custom_action must return a handled-bool"
);
assert!(
module.contains("self.0.custom_action(ctx, action)")
&& !module.contains("self.0.custom_action(ctx, action);"),
"the adapter must return custom_action's result, not discard it"
);
}
#[test]
fn typed_hook_mapping_skips_lexer_rule_predicates() {
// Combined grammar: a lexer-rule helper predicate precedes the
// parser-rule helper predicate. The typed-hook scan must skip the
// lexer-rule block so it does not consume the parser predicate
// coordinate and wire the adapter to the wrong method.
let combined = concat!(
"grammar S;\n",
"ID : {aheadIsDigit()}? [a-z]+ ;\n",
"s : {isTypeName()}? A ;\n",
);
let mappings = parser_typed_hook_mappings(
&predicate_parser_data(),
Some(combined),
&SemPatternFile::default(),
)
.expect("typed hook mapping should succeed");
// Only the parser-rule helper (`isTypeName` on rule `s`, pred 0) maps;
// the lexer-rule `aheadIsDigit` helper is not wired to a parser hook.
assert_eq!(
mappings.len(),
1,
"only the parser-rule helper maps: {mappings:?}"
);
assert_eq!((mappings[0].rule_index, mappings[0].pred_index), (0, 0));
assert_eq!(mappings[0].method_name, "is_type_name");
}
#[test]
fn typed_hook_predicate_method_name_avoids_action_hook_collision() {
// A grammar helper that normalizes to the reserved action-hook method
// name must be disambiguated, or the generated trait would declare two
// `custom_action` methods (Rust has no arity overloading).
assert_eq!(
typed_hook_predicate_method_name("customAction"),
"custom_action_pred"
);
assert_eq!(
typed_hook_predicate_method_name("custom_action"),
"custom_action_pred"
);
// An unrelated helper keeps its normalized name.
assert_eq!(
typed_hook_predicate_method_name("isTypeName"),
"is_type_name"
);
}
#[test]
fn typed_hook_adapter_disambiguates_custom_action_helper() {
// End-to-end: a bare `customAction()` predicate helper must not collide
// with the fixed `custom_action` action-hook method on the trait.
let grammar = "grammar S;\ns : {customAction()}? A ;\n";
let mappings = parser_typed_hook_mappings(
&predicate_parser_data(),
Some(grammar),
&SemPatternFile::default(),
)
.expect("typed hook mapping should succeed");
assert_eq!(mappings.len(), 1);
assert_eq!(mappings[0].method_name, "custom_action_pred");
let adapter = render_typed_hook_adapter("SParser", &mappings);
// The predicate method is the disambiguated name; the action hook keeps
// the reserved name — two distinct methods, so the trait compiles.
assert!(adapter.contains("fn custom_action_pred<L>"));
assert!(adapter.contains(
"fn custom_action<L>(&mut self, _ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>, _action: antlr4_runtime::ParserAction) -> bool"
));
assert!(adapter.contains("Some(self.0.custom_action_pred(ctx))"));
}
#[test]
fn manifest_predicate_provenance_skips_lexer_rule_block() {
// A combined grammar with a lexer-rule predicate before the parser-rule
// predicate: the manifest must attach the *parser* predicate's body/line
// to the parser coordinate, not the lexer predicate's (which would drift
// provenance under positional pairing).
let combined = concat!(
"grammar S;\n",
"ID : {aheadIsDigit()}? [a-z]+ ;\n",
"s : {isTypeName()}? A ;\n",
);
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(combined),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let predicate = entries
.iter()
.find(|entry| entry.kind == SemanticsKind::ParserPredicate)
.expect("parser predicate coordinate present");
assert_eq!(predicate.rule_name.as_deref(), Some("s"));
assert_eq!(
predicate.body.as_deref(),
Some("isTypeName()"),
"provenance must be the parser predicate body, not the lexer-rule aheadIsDigit()"
);
}
#[test]
fn require_full_semantics_rejects_policy_fallbacks_but_allows_hooks() {
let fallback = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
assert!(enforce_require_full_semantics(true, &fallback).is_err());
let mut hooked = fallback;
hooked[0].disposition = SemanticsDisposition::Hooked;
enforce_require_full_semantics(true, &hooked).expect("hooked coordinates are complete");
}
#[test]
fn collect_parser_semantics_inventories_untranslated_predicates() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
assert_eq!(entries.len(), 1);
let entry = &entries[0];
assert_eq!(entry.kind, SemanticsKind::ParserPredicate);
assert_eq!(entry.disposition, SemanticsDisposition::AssumeTrue);
assert_eq!(entry.rule_name.as_deref(), Some("s"));
assert_eq!(entry.rule_index, Some(0));
assert_eq!(entry.index, Some(0));
assert_eq!(entry.body.as_deref(), Some("isTypeName()"));
assert_eq!(entry.line, Some(2));
assert!(entry.template.is_none());
}
#[test]
fn collect_parser_semantics_marks_supported_predicates_translated() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some("parser grammar S;\ns : {true}? A ;\n"),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].disposition, SemanticsDisposition::Translated);
assert_eq!(entries[0].template.as_deref(), Some("True"));
}
#[test]
fn collect_parser_semantics_marks_helper_hooked_predicates_hooked() {
let patterns = parse_sem_patterns(
"version = 1\n\n[[helper]]\nname = \"isTypeName\"\nreturns = \"bool\"\nlower = \"hook\"\n",
)
.expect("pattern file should parse");
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::Error,
&patterns,
)
.expect("collection should succeed");
assert_eq!(entries.len(), 1);
// A helper routed to the hook trait is accounted for, but it is NOT a
// translation: the manifest must say `hooked` so users know the
// coordinate needs a runtime hook implementation.
assert_eq!(entries[0].disposition, SemanticsDisposition::Hooked);
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("hooked coordinates satisfy strict mode");
}
#[test]
fn collect_parser_semantics_without_grammar_source_keeps_coordinates() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
None,
SemUnknownPolicy::AssumeFalse,
&SemPatternFile::default(),
)
.expect("collection should succeed");
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].disposition, SemanticsDisposition::AssumeFalse);
assert!(entries[0].body.is_none());
assert!(entries[0].line.is_none());
}
#[test]
fn enforce_sem_unknown_error_lists_untranslated_coordinates() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let error = enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect_err("untranslated predicate must fail generation");
let message = error.to_string();
assert!(
message.contains("unsupported semantic predicate"),
"message should name the failure class: {message}"
);
assert!(message.contains("rule=s(0)"), "message: {message}");
assert!(message.contains("pred_index=0"), "message: {message}");
assert!(message.contains("at 2:4"), "message: {message}");
assert!(message.contains("{isTypeName()}"), "message: {message}");
assert!(
message.contains("--sem-unknown=error"),
"message: {message}"
);
}
#[test]
fn enforce_sem_unknown_error_accepts_fully_translated_grammars() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some("parser grammar S;\ns : {true}? A ;\n"),
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("collection should succeed");
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("fully translated grammar passes strict mode");
}
#[test]
fn authored_empty_parser_action_is_explicit_noop() {
let grammar = "parser grammar T;\ns : {} A ;\n";
let entries = collect_parser_semantics(
&action_parser_data(),
Some(grammar),
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let action = entries
.iter()
.find(|entry| entry.kind == SemanticsKind::ParserAction)
.expect("action entry is inventoried");
assert_eq!(action.atn_state, Some(0));
assert_eq!(action.body.as_deref(), Some(""));
assert_eq!(action.disposition, SemanticsDisposition::Synthetic);
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("empty authored action is a strict-policy no-op");
let module = render_parser_with_options(
"TParser",
&action_parser_data(),
Some(grammar),
ParserRenderOptions {
require_generated_parser: true,
embedded: false,
sem_unknown: SemUnknownPolicy::Error,
patterns: None,
},
)
.expect("empty action should not block generated parser output");
assert!(module.contains("0 => {}"));
}
#[test]
fn parser_action_attribution_ignores_non_embedded_member_syntax() {
let grammar = "parser grammar T;\n@members { int x; }\ns : { native(); } A ;\n";
let entries = collect_parser_semantics(
&action_parser_data(),
Some(grammar),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("Java-style members are irrelevant to action attribution");
let action = entries
.iter()
.find(|entry| entry.kind == SemanticsKind::ParserAction)
.expect("action entry is inventoried");
assert_eq!(action.atn_state, Some(0));
assert_eq!(action.body.as_deref(), Some("native();"));
}
#[test]
fn authored_action_block_counter_counts_only_authored_parser_blocks() {
// Counts author-written action `{...}` blocks per parser rule, including
// native/untranslated ones, but excludes predicates, `@init`/`@after`,
// members/options blocks, and rule exception handlers.
let grammar = "grammar T;\n\
@members { int shared; }\n\
s @init { init(); } : ID { native(); } { <writeln(\"x\")> } EOF ;\n\
catch [antlr4_runtime::AntlrError error] { recover(error); }\n\
finally { cleanup(); }\n\
t : {pred()}? ID ;\n\
ID : [a-z]+ ;\n";
let rule_names = vec!["s".to_owned(), "t".to_owned()];
let counts = authored_parser_action_blocks_per_rule(grammar, &rule_names);
// Rule s: two body actions ({native()} + {<writeln>}). The @init and
// @members blocks and exception handlers are excluded; the predicate on
// t is excluded.
assert_eq!(counts.get(&0).copied(), Some(2), "counts: {counts:?}");
assert_eq!(
counts.get(&1).copied(),
None,
"rule t has no action block: {counts:?}"
);
}
#[test]
fn enforce_sem_unknown_error_exempts_synthetic_actions() {
// A `Synthetic` action (ANTLR-inserted, e.g. LR elimination) must NOT
// fail under the Error policy: there is no author intent to implement.
let synthetic = SemanticsEntry {
kind: SemanticsKind::ParserAction,
rule_index: Some(1),
rule_name: Some("declarator".to_owned()),
index: None,
atn_state: Some(9),
line: None,
column: None,
body: None,
disposition: SemanticsDisposition::Synthetic,
template: None,
};
enforce_sem_unknown(SemUnknownPolicy::Error, std::slice::from_ref(&synthetic))
.expect("a synthetic action is exempt from the error gate");
// ...but an authored untranslated action (Ignored) at the same shape must fail.
let authored = SemanticsEntry {
disposition: SemanticsDisposition::Ignored,
..synthetic
};
enforce_sem_unknown(SemUnknownPolicy::Error, std::slice::from_ref(&authored))
.expect_err("an authored untranslated action must fail loud under error");
}
#[test]
fn enforce_sem_unknown_is_lenient_under_default_policy() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
enforce_sem_unknown(SemUnknownPolicy::AssumeTrue, &entries)
.expect("assume-true keeps the historical lenient behavior");
}
#[test]
fn enforce_sem_unknown_fails_per_coordinate_error_under_default_policy() {
// A per-coordinate `dispose = "error"` override must fail codegen even
// when the global policy is the lenient default; otherwise the
// coordinate lowers to no SemIR entry and silently falls back to
// AssumeTrue at runtime.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"error\"\n",
)
.expect("pattern file parses");
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::AssumeTrue,
&patterns,
)
.expect("collection should succeed");
let error = enforce_sem_unknown(SemUnknownPolicy::AssumeTrue, &entries)
.expect_err("per-coordinate error override must fail even under assume-true");
assert!(
error.to_string().contains("pred_index=0"),
"message should name the rejected coordinate: {error}"
);
}
#[test]
fn semantics_manifest_renders_coordinates_and_policy() {
let entries = collect_parser_semantics(
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
)
.expect("collection should succeed");
let manifest = render_semantics_manifest(
SemUnknownPolicy::AssumeTrue,
&[],
&[("parser", "SParser".to_owned(), entries)],
);
assert!(manifest.contains("\"version\": 2"));
assert!(manifest.contains("\"options\": []"));
assert!(manifest.contains("\"policy\": \"assume-true\""));
assert!(manifest.contains("\"kind\": \"parser\""));
assert!(manifest.contains("\"name\": \"SParser\""));
assert!(manifest.contains("\"kind\": \"parser-predicate\""));
assert!(manifest.contains("\"rule\": \"s\""));
assert!(manifest.contains("\"disposition\": \"assume-true\""));
assert!(manifest.contains("\"body\": \"isTypeName()\""));
assert!(manifest.contains("\"line\": 2"));
}
#[test]
fn translates_portable_boolean_local_semantics() {
let data = portable_bool_parser_data();
let portable =
build_portable_local_data(&data, PORTABLE_BOOL_GRAMMAR, &SemPatternFile::default())
.expect("portable local semantics should build");
assert_eq!(
portable.declarations,
[vec!["let mut __antlr_local_seen = false;".to_owned()]]
);
assert_eq!(portable.required_generated_rules, BTreeSet::from([0]));
assert_eq!(
portable.inline_actions.get(&0).map(String::as_str),
Some("__antlr_local_seen = true;")
);
assert_eq!(
portable
.predicates
.get(&(0, 0))
.map(|(condition, message)| (condition.as_str(), message.as_deref())),
Some(("__antlr_local_seen", Some("not seen")))
);
let module = render_parser("SParser", &data, Some(PORTABLE_BOOL_GRAMMAR))
.expect("portable grammar should render");
assert!(module.contains("let mut __antlr_local_seen = false;"));
assert!(module.contains("__antlr_local_seen = true;"));
assert!(module.contains("if !(__antlr_local_seen) {"));
assert!(module.contains("failed_predicate_option_error(0, \"not seen\".to_owned())"));
let entries = collect_parser_semantics(
&data,
Some(PORTABLE_BOOL_GRAMMAR),
SemUnknownPolicy::Error,
&SemPatternFile::default(),
)
.expect("portable coordinates should be inventoried");
assert_eq!(entries.len(), 2);
assert!(
entries
.iter()
.all(|entry| entry.disposition == SemanticsDisposition::Translated)
);
assert!(
entries
.iter()
.all(|entry| entry.template.as_deref() == Some("PortableBooleanLocal"))
);
enforce_sem_unknown(SemUnknownPolicy::Error, &entries)
.expect("portable coordinates satisfy strict semantics");
}
#[test]
fn semantics_manifest_renders_empty_inventory() {
let manifest = render_semantics_manifest(
SemUnknownPolicy::AssumeTrue,
&[],
&[("parser", "SParser".to_owned(), Vec::new())],
);
assert!(manifest.contains("\"coordinates\": []"));
}
#[test]
fn assume_false_policy_reaches_generated_runtime_options() {
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::AssumeFalse,
patterns: None,
},
)
.expect("parser should render");
assert!(module.contains(
"unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::AssumeFalse"
));
}
#[test]
fn generated_top_level_entry_surfaces_unknown_semantic_error() {
// The public generated entry must surface Error-policy coordinates the
// generated-direct predicate path recorded, or a parse that consulted an
// unimplemented hook predicate returns a recovered Ok tree instead of
// AntlrError::Unsupported.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::AssumeFalse,
patterns: None,
},
)
.expect("parser should render");
let surface_at = module
.find("if let Some(error) = self.base.take_unknown_semantic_error()")
.expect("generated top-level entry must surface recorded unknown-semantic coordinates");
// Guarded by the public entry only, not the nested (from-generated) path.
assert!(module.contains("if allow_generated_fallback {"));
// The fail-loud check must run before the generated entry can return Ok.
let ok_at = module[surface_at..]
.find("Ok(__tree)")
.map(|offset| surface_at + offset)
.expect("generated entry returns the tree after semantic checks");
assert!(surface_at < ok_at);
}
#[test]
fn generated_rule_error_prefers_recorded_semantic_error() {
// When a generated-direct predicate consulted an unimplemented hook
// (returning None under the Error policy), the alternative fails and
// `parse_generated_rule` returns a generic `failed_predicate_error`. The
// top-level `Err` arm must first drain any recorded fail-loud coordinate
// and return that `AntlrError::Unsupported`, otherwise the documented
// fail-loud error is shadowed by the generic rule error. The check is
// gated on `allow_generated_fallback` so a nested child keeps its hits for
// the generated parent to surface at its own boundary.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::Hook,
patterns: None,
},
)
.expect("parser should render");
// Locate the generated-rule `Err` arm's generic return.
let generic_return_at = module
.find("return Err(error.into_error());")
.expect("generated-rule Err arm returns the generic rule error");
// The fail-loud drain must appear inside that arm, before the generic
// return, under the top-level gate.
let arm_start = module[..generic_return_at]
.rfind("Err(error) => {")
.expect("generic return lives in the Err arm");
let arm = &module[arm_start..generic_return_at];
assert!(
arm.contains("if allow_generated_fallback {")
&& arm.contains(
"if let Some(semantic_error) = self.base.take_unknown_semantic_error()"
),
"the Err arm must drain a recorded semantic error before the generic return"
);
}
#[test]
fn interpreted_fallback_action_miss_is_surfaced_at_public_entry() {
// A public entry that falls back to the interpreted ATN path runs the
// non-buffered `run_action` loop immediately (an untranslated action
// routed to `parser_action_hook` records an `unhandled_action_hit` under
// the Error policy). The top-level surfacing check that drains those hits
// is gated on the SAME `allow_generated_fallback` condition as the branch
// that runs the interpreted fallback, so the miss cannot escape as `Ok`.
// (Verified end-to-end: parsing an untranslated action through the
// interpreted path under `--sem-unknown=hook` with a declining hook
// returns `AntlrError::Unsupported("unhandled semantic action: ...")`.)
//
// Emitting a *separate* check inside `parse_interpreted_rule_precedence`
// would be both dead (the outer check already drains the hits) and
// unsafe: an early `return Err` there would bypass the caller's ordinary
// generated-rule cleanup and error reporting path.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::Hook,
patterns: None,
},
)
.expect("parser should render");
// The interpreted call site in the top-level entry and the surfacing check
// share the `allow_generated_fallback` gate, so the surfacing check follows
// the interpreted call and drains any action-hook miss (or predicate miss)
// the fallback recorded.
let interpreted_call_at = module
.find("self.parse_interpreted_rule_precedence(rule_index, precedence)?")
.expect("top-level entry runs the interpreted fallback under allow_generated_fallback");
let surface_at = module[interpreted_call_at..]
.find("if let Some(error) = self.base.take_unknown_semantic_error()")
.map(|offset| interpreted_call_at + offset)
.expect("the public entry must drain recorded semantic misses after the fallback");
// Between the interpreted fallback call and the surfacing check the entry
// must not return `Ok`, or an action-hook miss recorded by the immediate
// `run_action` loop would escape as a recovered success.
assert!(
!module[interpreted_call_at..surface_at].contains("Ok(__tree)"),
"the entry must not return Ok between the interpreted fallback and the surfacing check"
);
// Both are under the same gate: the branch that runs the fallback and the
// check that drains its misses share `if allow_generated_fallback {`.
assert!(
module[..interpreted_call_at].contains("} else {"),
"the interpreted fallback remains the non-generated fallback path"
);
}
#[test]
fn hook_predicate_does_not_escalate_default_policy() {
// A per-coordinate `dispose = "hook"` predicate under the default global
// policy must NOT flip the whole parser to Error — that would turn
// unrelated `assume-true` coordinates into fail-loud. The hook falls
// through to the configured (default) policy per coordinate; users opt
// into fail-loud with --sem-unknown=error.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: SemUnknownPolicy::AssumeTrue,
patterns: Some(&patterns),
},
)
.expect("parser should render");
assert!(
!module.contains("UnknownSemanticPolicy::Error"),
"a hook predicate must not escalate the default policy to Error"
);
}
#[test]
fn non_default_policy_installs_on_generated_parser_constructor() {
// The generated-direct predicate path reads BaseParser's
// `unknown_predicate_policy`, which the interpreter options never set on
// that path. The constructor must install a non-default policy so a
// generated rule's hook predicate honors --sem-unknown instead of the
// AssumeTrue default.
for (policy, literal) in [
(
SemUnknownPolicy::AssumeFalse,
"antlr4_runtime::UnknownSemanticPolicy::AssumeFalse",
),
(
SemUnknownPolicy::Error,
"antlr4_runtime::UnknownSemanticPolicy::Error",
),
] {
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: policy,
patterns: None,
},
)
.expect("parser should render");
assert!(
module.contains(&format!("base.set_unknown_predicate_policy({literal});")),
"policy {policy:?} must be installed on the generated constructor"
);
}
// The default policy leaves the constructor untouched (no needless call).
let default_module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some(PREDICATE_GRAMMAR),
ParserRenderOptions::default(),
)
.expect("parser should render");
assert!(!default_module.contains("set_unknown_predicate_policy"));
}
#[test]
fn default_policy_emits_assume_true_options_field() {
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
Some("parser grammar S;\ns : {true}? A ;\n"),
ParserRenderOptions::default(),
)
.expect("parser should render");
assert!(module.contains(
"unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::AssumeTrue"
));
}
#[test]
fn non_default_policy_disables_adaptive_direct_gate() {
// A grammar with no predicates/actions would normally allow the
// adaptive-direct shortcut, but that path drops the emitted
// ParserRuntimeOptions (and thus the policy). The gate must be disabled
// so a non-default policy always reaches the options-carrying call.
let default_module = render_parser_with_options("TParser", &minimal_parser_data(), None, {
ParserRenderOptions::default()
})
.expect("parser should render under default policy");
assert!(
default_module.contains("&& true && std::env::var_os(\"ANTLR4_RUST_ADAPTIVE_DIRECT\")"),
"the predicate-free fixture must allow adaptive-direct by default, or this test proves nothing"
);
for policy in [SemUnknownPolicy::AssumeFalse, SemUnknownPolicy::Error] {
let module = render_parser_with_options(
"TParser",
&minimal_parser_data(),
None,
ParserRenderOptions {
require_generated_parser: false,
embedded: false,
sem_unknown: policy,
patterns: None,
},
)
.expect("parser should render under a non-default policy");
assert!(
module.contains("&& false && std::env::var_os(\"ANTLR4_RUST_ADAPTIVE_DIRECT\")"),
"policy {policy:?} must disable the adaptive-direct gate"
);
}
}
#[test]
fn lexer_assume_false_policy_renders_failing_predicate_hook() {
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
None,
false,
SemUnknownPolicy::AssumeFalse,
&SemPatternFile::default(),
false,
)
.expect("lexer should render");
assert!(module.contains("next_token_compiled_with_hooks"));
assert!(module.contains("|_, _| false"));
}
#[test]
fn lexer_per_coordinate_hook_override_routes_to_owned_hooks() {
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"lexer-predicate\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
None,
false,
SemUnknownPolicy::AssumeTrue,
&patterns,
false,
)
.expect("per-coordinate hooks should render generated lexer plumbing");
assert!(module.contains("next_token_compiled_with_semantic_dispatch"));
assert!(module.contains("=> { None }"));
}
#[test]
fn lexer_per_coordinate_assume_false_override_renders_failing_arm() {
// A per-coordinate `dispose = "assume-false"` on an uncovered lexer
// predicate must render an explicit failing `run_predicate` arm and take
// the hook-taking token path even under the default global policy, so
// the override recorded in the manifest actually removes the guarded
// alternative at runtime.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"lexer-predicate\"\nindex = 0\ndispose = \"assume-false\"\n",
)
.expect("pattern file parses");
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
None,
false,
SemUnknownPolicy::AssumeTrue,
&patterns,
false,
)
.expect("lexer should render");
// The uncovered coordinate now has an explicit `false` predicate arm and
// the lexer takes the hook-carrying token path (not next_token_compiled).
assert!(
module.contains("=> { Some(false) }"),
"override renders a failing arm"
);
assert!(module.contains("run_predicate"));
assert!(!module.contains("next_token_compiled(&mut self.base, sink, atn(), lexer_dfa())"));
}
#[test]
fn coordinate_override_applies_without_grammar_source() {
// A `--sem-patterns` coordinate override resolves by ATN coordinate, so
// it must take effect in the generated parser even without --grammar,
// rather than being reported active in the manifest yet silently
// dropped in generated code.
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"predicate\"\nrule = \"s\"\nindex = 0\ndispose = \"assume-false\"\n",
)
.expect("pattern file parses");
let templates =
parser_predicate_templates_from_overrides(&predicate_parser_data(), &patterns)
.expect("override synthesis should succeed");
assert_eq!(templates, [((0, 0), PredicateTemplate::False)]);
// And the rendered parser without grammar source carries the SemIR
// predicate for that coordinate.
let module = render_parser_with_options(
"SParser",
&predicate_parser_data(),
None,
ParserRenderOptions {
patterns: Some(&patterns),
..ParserRenderOptions::default()
},
)
.expect("parser should render");
assert!(
module.contains("rule_index: 0, pred_index: 0"),
"override-derived predicate must reach parser_semantics() without --grammar"
);
}
#[test]
fn lexer_hook_policy_routes_uncovered_predicates_to_owned_hooks() {
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
None,
false,
SemUnknownPolicy::Hook,
&SemPatternFile::default(),
false,
)
.expect("hook policy should render generated hook plumbing");
assert!(module.contains("next_token_compiled_with_semantic_dispatch"));
assert!(module.contains("hooks: H"));
}
#[test]
fn hook_disposed_lexer_actions_require_semantic_dispatch() {
let coordinates = [(0, 0)];
let rule_names = ["A".to_owned()];
let no_actions = [];
assert!(lexer_actions_require_semantic_hooks(
&coordinates,
&rule_names,
&no_actions,
&SemPatternFile::default(),
SemUnknownPolicy::Hook,
));
let patterns = parse_sem_patterns(
"version = 1\n[[coordinate]]\nkind = \"lexer-action\"\nindex = 0\ndispose = \"hook\"\n",
)
.expect("pattern file parses");
assert!(lexer_actions_require_semantic_hooks(
&coordinates,
&rule_names,
&no_actions,
&patterns,
SemUnknownPolicy::AssumeTrue,
));
let translated = [((0, 0), ActionTemplate::LexerPopMode)];
assert!(!lexer_actions_require_semantic_hooks(
&coordinates,
&rule_names,
&translated,
&SemPatternFile::default(),
SemUnknownPolicy::Hook,
));
}
#[test]
fn lexer_hook_disposition_is_full_semantics() {
let entry = SemanticsEntry {
kind: SemanticsKind::LexerAction,
rule_index: Some(0),
rule_name: Some("A".to_owned()),
index: Some(0),
atn_state: None,
line: None,
column: None,
body: Some("this.handle();".to_owned()),
disposition: SemanticsDisposition::Hooked,
template: None,
};
enforce_require_full_semantics(true, &[entry])
.expect("hooked lexer actions are implemented by generated hook plumbing");
}
#[test]
fn lexer_default_policy_keeps_compiled_token_path() {
let module = render_lexer(
"SLexer",
&predicate_lexer_data(),
None,
false,
SemUnknownPolicy::AssumeTrue,
&SemPatternFile::default(),
false,
)
.expect("lexer should render");
assert!(module.contains("next_token_compiled(&mut self.base, sink, atn(), lexer_dfa())"));
assert!(module.contains("if H::ENABLES_LEXER_LIFECYCLE"));
assert!(module.contains("next_token_compiled_with_semantic_dispatch"));
assert!(module.contains("pub fn reset(&mut self)"));
assert!(module.contains("reset_with_semantic_hooks"));
assert!(module.contains("pub fn set_input_stream(&mut self, input: I)"));
assert!(module.contains("set_input_stream_with_semantic_hooks"));
assert!(module.contains("self.base.set_input_stream(input)"));
assert!(module.contains("pub fn clear_dfa(&self)"));
assert!(module.contains("self.base.clear_dfa()"));
assert!(module.contains("pub fn add_error_listener<T>(&mut self, listener: T)"));
assert!(module.contains(
"T: for<'a> antlr4_runtime::ErrorListener<dyn antlr4_runtime::Recognizer + 'a> + Send + 'static,"
));
assert!(module.contains("self.base.add_error_listener(listener)"));
assert!(module.contains("pub fn remove_error_listeners(&mut self)"));
assert!(module.contains("self.base.remove_error_listeners()"));
assert!(module.contains(
"fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result<TokenId, TokenStoreError>"
));
assert!(module.contains(
"fn source_text(&self) -> Option<std::rc::Rc<str>> { self.base.source_text() }"
));
assert!(module.contains(
"fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool"
));
assert!(module.contains("Recognizer::notify_error_listeners("));
assert!(!module.contains("CommonToken"));
assert!(!module.contains("TokenFactory"));
}
#[test]
fn grammar_options_inventory_distinguishes_metadata_and_target_behavior() {
let source = "\
lexer grammar L;
options {
tokenVocab = Shared;
caseInsensitive = true;
superClass = BaseLexer;
}
A options { caseInsensitive = false; } : 'a';
";
let options_open = templates::find_significant_open_brace(source, 0)
.expect("grammar options block should be structurally visible");
assert!(is_options_block(source, options_open));
let statement_start = statement_start_before(source, options_open);
assert_eq!(
rule_header_start_identifier(&source[statement_start..options_open]),
Some("options")
);
let options = collect_grammar_options(source, &BTreeSet::new())
.expect("top-level grammar options should parse");
assert_eq!(
options,
[
GrammarOptionEntry {
key: "tokenVocab".to_owned(),
value: "Shared".to_owned(),
line: 3,
column: 2,
disposition: GrammarOptionDisposition::Metadata,
},
GrammarOptionEntry {
key: "caseInsensitive".to_owned(),
value: "true".to_owned(),
line: 4,
column: 2,
disposition: GrammarOptionDisposition::Metadata,
},
GrammarOptionEntry {
key: "superClass".to_owned(),
value: "BaseLexer".to_owned(),
line: 5,
column: 2,
disposition: GrammarOptionDisposition::Unsupported,
},
]
);
}
#[test]
fn grammar_option_hook_acknowledgement_satisfies_strict_mode() {
let source = "lexer grammar L;\noptions { superClass = BaseLexer; }\nA: 'a';\n";
let hooks = BTreeSet::from(["superClass=BaseLexer".to_owned()]);
let options =
collect_grammar_options(source, &hooks).expect("acknowledged option should parse");
assert_eq!(options.len(), 1);
assert_eq!(options[0].disposition, GrammarOptionDisposition::Hooked);
enforce_require_full_options(true, &options)
.expect("hooked options are explicitly implemented");
let manifest = render_semantics_manifest(SemUnknownPolicy::AssumeTrue, &options, &[]);
assert!(manifest.contains("\"name\": \"superClass\""));
assert!(manifest.contains("\"value\": \"BaseLexer\""));
assert!(manifest.contains("\"disposition\": \"hooked\""));
}
#[test]
fn strict_mode_rejects_unacknowledged_target_options() {
let source = "parser grammar P;\noptions { contextSuperClass = RuleNode; }\ns: EOF;\n";
let options = collect_grammar_options(source, &BTreeSet::new())
.expect("unsupported option should still be inventoried");
let error = enforce_require_full_options(true, &options)
.expect_err("strict mode should reject an unsupported target option");
assert!(
error
.to_string()
.contains("unsupported grammar option: contextSuperClass=RuleNode at 2:10")
);
assert!(error.to_string().contains("--option-hook KEY=VALUE"));
}
#[test]
fn option_hook_requires_an_exact_assignment() {
assert_eq!(
normalize_option_hook(" superClass = BaseLexer ")
.expect("valid hook assignment should normalize"),
"superClass=BaseLexer"
);
assert!(normalize_option_hook("superClass").is_err());
assert!(normalize_option_hook("=BaseLexer").is_err());
assert!(normalize_option_hook("superClass=").is_err());
}
#[test]
fn lexer_run_predicate_default_arm_follows_policy() {
// A mixed lexer (one covered coordinate plus an uncovered one that
// lands on the catch-all arm) must honor `--sem-unknown`: assume-false
// rejects the uncovered predicate instead of leaving it viable.
let predicates = [((0_usize, 0_usize), PredicateTemplate::True)];
let assume_true = render_lexer_predicate_method(&predicates, SemUnknownPolicy::AssumeTrue);
assert!(assume_true.contains("_ => Some(true),"));
assert!(!assume_true.contains("_ => Some(false),"));
let assume_false =
render_lexer_predicate_method(&predicates, SemUnknownPolicy::AssumeFalse);
assert!(assume_false.contains("_ => Some(false),"));
assert!(!assume_false.contains("_ => Some(true),"));
// `error`/`hook` keep the conservative assume-true lexer default that
// matches historical behavior (lexer fail-loud is a codegen-time error,
// not a runtime catch-all).
let error = render_lexer_predicate_method(&predicates, SemUnknownPolicy::Error);
assert!(error.contains("_ => Some(true),"));
}
}