antlr-rust-codegen 0.33.0

ANTLR v4 grammar compiler and Rust source generator
Documentation
// SPDX-License-Identifier: BSD-3-Clause
// Copyright (c) 2026 Konstantin Vyatkin
pub(crate) mod prelude {
    pub(crate) use super::{
        GENERATED_MODULE_FOOTER, LexerCodegenData, ParserCodegenData, RecognizerCodegenData,
        allocate_const_name, generated_module_header, max_len, render_recognizer_token_constants,
        token_type_for_name,
    };
    pub(crate) use std::collections::{BTreeMap, BTreeSet, btree_map::Entry};
    pub(crate) use std::env;
    pub(crate) use std::fmt::Write as _;
    pub(crate) use std::fs;
    pub(crate) use std::io;
    pub(crate) use std::ops::AddAssign;
    pub(crate) use std::path::{Path, PathBuf};

    pub(crate) use antlr4_runtime::atn::lexer_dfa::CompiledLexerDfa;
    pub(crate) use antlr4_runtime::atn::parser_atn::{
        ParserAtn, ParserAtnState, ParserTokenSetKind, ParserTransition, ParserTransitionData,
    };
    pub(crate) use antlr4_runtime::atn::{AtnStateKind, LexerAction, LexerAtn, LexerTransition};
    pub(crate) use antlr4_runtime::token::TOKEN_EOF;
    pub(crate) use petgraph::graph::DiGraph;
    pub(crate) use petgraph::visit::{Dfs, Reversed};

    pub(crate) use crate::artifact::{GeneratedArtifacts, Generation};
    pub(crate) use crate::embedded;
    pub(crate) use crate::error::{Diagnostic, Error, Severity};
    pub(crate) use crate::grammar;
    pub(crate) use crate::grammar::atn::{
        CompiledLexer, CompiledParser, FinalizedAtnGraph, FinalizedTransitionKind,
    };
    pub(crate) use crate::grammar::frontend::{SourceId, SourceSpan};
    pub(crate) use crate::grammar::integration::source_implicit_token_literals;
    pub(crate) use crate::grammar::loader::LoadOptions;
    pub(crate) use crate::grammar::model::{
        Alternative, AlternativeId, AttributeSymbol, Block, Element, ElementKind, GrammarKind,
        LabelKind, LeftRecursiveAlternativeKind, ModelNodeId, Quantifier, Rule, SemanticGrammar,
        SetElement, Terminal, Vocabulary,
    };
    pub(crate) use crate::grammar::provenance::{Origin, ProvenanceIndex};
    pub(crate) use crate::grammar::source::SourceSet;
    #[cfg(test)]
    pub(crate) use crate::rust_output::is_rust_keyword;
    pub(crate) use crate::rust_output::{
        module_name, replace_all, rust_function_name, rust_identifier, rust_string, rust_type_name,
        sanitize_identifier, split_identifier_words,
    };
}

use prelude::*;

pub(crate) fn generated_module_header() -> String {
    format!(
        concat!(
            "// @generated by ",
            env!("CARGO_PKG_NAME"),
            " v",
            env!("CARGO_PKG_VERSION"),
            " - do not edit\n",
            "// project: ",
            env!("CARGO_PKG_REPOSITORY"),
            "\n",
            "antlr4_runtime::__antlr4_rust_require_codegen_api!({}, \"{}\");\n",
            "#[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]\n",
            "#[rustfmt::skip]\n",
            "mod __antlr4_rust_generated {{\n",
            "\n",
        ),
        antlr4_runtime::__ANTLR4_RUST_CODEGEN_API,
        env!("CARGO_PKG_VERSION")
    )
}

pub(crate) const GENERATED_MODULE_FOOTER: &str = "\
}

#[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]
pub use self::__antlr4_rust_generated::*;
";

#[derive(Clone, Debug, Default)]
pub(crate) struct RecognizerCodegenData<'a> {
    pub(crate) literal_names: Vec<Option<String>>,
    pub(crate) symbolic_names: Vec<Option<String>>,
    pub(crate) rule_names: Vec<String>,
    pub(crate) semantic: Option<&'a SemanticGrammar>,
    pub(crate) graph: Option<&'a FinalizedAtnGraph>,
    pub(crate) provenance: Option<&'a ProvenanceIndex>,
    pub(crate) sources: Option<&'a SourceSet>,
}

#[derive(Clone, Debug)]
pub(crate) struct LexerCodegenData<'a> {
    pub(crate) common: RecognizerCodegenData<'a>,
    pub(crate) channel_names: Vec<String>,
    pub(crate) channel_numbers: BTreeMap<String, i32>,
    pub(crate) mode_names: Vec<String>,
    pub(crate) mode_numbers: BTreeMap<String, usize>,
    pub(crate) lexer_atn_words: Vec<i32>,
    pub(crate) lexer_atn: LexerAtn,
    pub(crate) lexer_dfa_words: Vec<u32>,
}

#[derive(Clone, Debug)]
pub(crate) struct ParserCodegenData<'a> {
    pub(crate) common: RecognizerCodegenData<'a>,
    pub(crate) parser_atn: ParserAtn,
}

impl<'a> LexerCodegenData<'a> {
    pub(crate) fn from_compiled(compiled: &'a CompiledLexer, sources: &'a SourceSet) -> Self {
        let recognizer = &compiled.semantic.recognizer;
        Self {
            common: RecognizerCodegenData {
                literal_names: recognizer.literal_names.clone(),
                symbolic_names: recognizer.symbolic_names.clone(),
                rule_names: recognizer.rule_names.clone(),
                semantic: Some(&compiled.semantic),
                graph: Some(&compiled.graph),
                provenance: Some(&compiled.provenance),
                sources: Some(sources),
            },
            channel_names: runtime_channel_names(&recognizer.channel_numbers),
            channel_numbers: recognizer.channel_numbers.clone(),
            mode_names: recognizer.mode_names.clone(),
            mode_numbers: recognizer.mode_numbers.clone(),
            lexer_atn_words: compiled.runtime_artifact.atn_words.clone(),
            lexer_atn: compiled.atn.clone(),
            lexer_dfa_words: compiled.runtime_artifact.dfa_words.clone(),
        }
    }

    pub(crate) const fn lexer_atn(&self) -> &LexerAtn {
        &self.lexer_atn
    }
}

impl<'a> ParserCodegenData<'a> {
    pub(crate) fn from_compiled(compiled: &'a CompiledParser, sources: &'a SourceSet) -> Self {
        let recognizer = &compiled.semantic.recognizer;
        Self {
            common: RecognizerCodegenData {
                literal_names: recognizer.literal_names.clone(),
                symbolic_names: recognizer.symbolic_names.clone(),
                rule_names: recognizer.rule_names.clone(),
                semantic: Some(&compiled.semantic),
                graph: Some(&compiled.graph),
                provenance: Some(&compiled.provenance),
                sources: Some(sources),
            },
            parser_atn: compiled.packed.clone(),
        }
    }

    pub(crate) const fn parser_atn(&self) -> &ParserAtn {
        &self.parser_atn
    }
}

impl<'a> std::ops::Deref for LexerCodegenData<'a> {
    type Target = RecognizerCodegenData<'a>;

    fn deref(&self) -> &Self::Target {
        &self.common
    }
}

impl<'a> std::ops::Deref for ParserCodegenData<'a> {
    type Target = RecognizerCodegenData<'a>;

    fn deref(&self) -> &Self::Target {
        &self.common
    }
}

/// Dense channel-name table indexed by runtime channel value, matching the
/// Java target's generated `channelNames` array. The semantic model keeps the
/// `.interp`-shaped vector (two `null` placeholder rows before user channels,
/// a tool serialization quirk), so rebuild from the value map instead of
/// copying it — otherwise a custom channel's display name lands at
/// `value + 2` and `channel_names[value]` reads as empty.
pub(crate) fn runtime_channel_names(channel_numbers: &BTreeMap<String, i32>) -> Vec<String> {
    let mut names = vec![String::new(); channel_numbers.len()];
    for (name, number) in channel_numbers {
        let index = usize::try_from(*number).expect("channel value is non-negative");
        if index >= names.len() {
            names.resize(index + 1, String::new());
        }
        names[index].clone_from(name);
    }
    names
}

pub(crate) fn token_type_for_name(
    data: &RecognizerCodegenData<'_>,
    token_name: &str,
) -> Option<usize> {
    data.symbolic_names
        .iter()
        .position(|name| name.as_deref() == Some(token_name))
}

/// Converts an ANTLR token/rule name into an upper-snake Rust constant name.
pub(crate) fn rust_const_name(name: &str) -> String {
    let words = split_identifier_words(name);
    let ident = if words.is_empty() {
        "TOKEN".to_owned()
    } else {
        ascii_uppercase(&words.join("_"))
    };
    sanitize_identifier(&ident)
}

/// Allocates a collision-free generated-constant identifier: `prefix`
/// followed by the upper-snake [`rust_const_name`] mangling of `name`.
///
/// The mangling is lossy — `compilationUnit` and `compilation_unit` both
/// yield `COMPILATION_UNIT` — and every constant a generated recognizer
/// module emits (token, `RULE_*`, `CHANNEL_*`, `MODE_*`) shares one Rust
/// value namespace, so callers thread a single `used` set per generated
/// module through every constant renderer (tokens first, matching emission
/// order). The first emission keeps the canonical identifier and later
/// colliders receive a deterministic `_2`, `_3`, ... suffix, mirroring the
/// numbered dedup used for generated rule methods and context types.
pub(crate) fn allocate_const_name(prefix: &str, name: &str, used: &mut BTreeSet<String>) -> String {
    let canonical = format!("{prefix}{}", rust_const_name(name));
    if used.insert(canonical.clone()) {
        return canonical;
    }
    let mut suffix = 2_usize;
    loop {
        let candidate = format!("{canonical}_{suffix}");
        if used.insert(candidate.clone()) {
            return candidate;
        }
        suffix += 1;
    }
}

/// Renders token constants from the vocabulary's implicit `T__n` tokens and
/// the symbolic token names, shared verbatim by lexer and parser emission.
///
/// Distinct token types whose names mangle to one identifier are
/// deduplicated with a numbered suffix instead of silently losing their
/// constant (a token mangling to the predefined `EOF` becomes `EOF_2`).
/// Every allocated identifier is recorded in `used` so the module's other
/// constant renderers cannot collide with a token constant.
pub(crate) fn render_recognizer_token_constants(
    data: &RecognizerCodegenData<'_>,
    used: &mut BTreeSet<String>,
) -> String {
    let mut out = String::from("pub const EOF: i32 = antlr4_runtime::TOKEN_EOF;\n");
    let _ = used.insert("EOF".to_owned());
    if let Some(semantic) = data.semantic {
        let vocabulary = &semantic.recognizer.vocabulary;
        for name in vocabulary
            .name_order
            .iter()
            .filter(|name| name.starts_with("T__"))
        {
            let ident = sanitize_identifier(name);
            let token_type = vocabulary.by_name[name];
            let _ = used.insert(ident.clone());
            writeln!(out, "pub const {ident}: i32 = {token_type};")
                .expect("writing to a string cannot fail");
        }
    }
    for (index, name) in data.symbolic_names.iter().enumerate() {
        let Some(name) = name else { continue };
        let ident = allocate_const_name("", name, used);
        writeln!(out, "pub const {ident}: i32 = {index};")
            .expect("writing to a string cannot fail");
    }
    out
}

/// Converts ASCII letters to upper case without using allocation-hiding string
/// case helpers disallowed by the strict Clippy policy.
pub(crate) fn ascii_uppercase(value: &str) -> String {
    value.chars().map(|ch| ch.to_ascii_uppercase()).collect()
}

pub(crate) fn max_len(left: &[Option<String>], right: &[Option<String>]) -> usize {
    left.len().max(right.len())
}

#[cfg(test)]
use crate::lexer::*;
#[cfg(test)]
use crate::parser::*;
#[cfg(test)]
use crate::semantics::*;
#[cfg(test)]
use crate::structural::*;

#[cfg(test)]
#[path = "generator/tests.rs"]
mod tests;