Skip to main content

dbt_antlr_codegen/
config.rs

1// SPDX-License-Identifier: BSD-3-Clause
2// Copyright (c) 2026 Bo Lin
3//! Build-script entry point: generate recognizers during `cargo build`, in
4//! the style of `prost-build` or `lalrpop`.
5
6use std::path::PathBuf;
7
8use crate::OptimizationOptions;
9#[cfg(feature = "generator")]
10use crate::emission::emit_files_with_options;
11use crate::error::{Diagnostic, EmitError, Severity};
12
13/// Generator configuration, primarily for use from a `build.rs` build script.
14///
15/// With the generator as a build dependency, no separate tool installation is
16/// needed and the generator version is pinned by `Cargo.lock` like any other
17/// dependency:
18///
19/// ```toml
20/// [dependencies]
21/// dbt-antlr-runtime = "0.1"
22///
23/// [build-dependencies]
24/// # default-features = false skips the `fancy` diagnostics rendering the
25/// # command-line tool uses; it shrinks the build-dependency tree by ~40%.
26/// dbt-antlr-codegen = { version = "0.1", default-features = false, features = ["generator"] }
27/// ```
28///
29/// ```rust,no_run
30/// // build.rs
31/// dbt_antlr_codegen::Config::new("grammars/Expr.g4")
32///     .generate()
33///     .unwrap_or_else(|error| panic!("{error:?}"));
34/// ```
35///
36/// Generated files land in the [output directory](Config::out_dir), which
37/// defaults to the `OUT_DIR` environment variable cargo sets for build
38/// scripts; pull them into the crate with `include!`:
39///
40/// ```rust,ignore
41/// mod expr {
42///     #![allow(clippy::all)]
43///     include!(concat!(env!("OUT_DIR"), "/exprparser.rs"));
44/// }
45/// ```
46///
47/// [`generate`](Config::generate) prints `cargo:rerun-if-changed` directives
48/// for the configured grammars and library directories (including each root
49/// grammar's own directory, which the loader searches for imported grammars),
50/// so the build script reruns exactly when grammar inputs change. It also
51/// prints each compiler diagnostic as a `cargo:warning` directive, so that
52/// cargo shows the warnings of the grammar (for example `G4S030` for a token
53/// that the lexer never emits) in the build output. Call
54/// [`quiet`](Config::quiet) to print only the errors.
55#[derive(Clone, Debug)]
56pub struct Config {
57    grammars: Vec<PathBuf>,
58    out_dir: Option<PathBuf>,
59    lib_dirs: Vec<PathBuf>,
60    gen_listener: bool,
61    gen_visitor: bool,
62    cargo_directives: bool,
63    quiet: bool,
64    optimization: OptimizationOptions,
65    pinned_release_version: Option<String>,
66    pinned_release_sha256: Option<String>,
67    pinned_release_base_url: Option<String>,
68}
69
70impl Config {
71    /// Starts a configuration generating from `grammar`.
72    pub fn new(grammar: impl Into<PathBuf>) -> Self {
73        Self {
74            grammars: vec![grammar.into()],
75            out_dir: None,
76            lib_dirs: Vec::new(),
77            gen_listener: true,
78            gen_visitor: false,
79            cargo_directives: true,
80            quiet: false,
81            optimization: OptimizationOptions::default(),
82            pinned_release_version: None,
83            pinned_release_sha256: None,
84            pinned_release_base_url: None,
85        }
86    }
87
88    /// Adds another root grammar to generate from.
89    pub fn grammar(&mut self, grammar: impl Into<PathBuf>) -> &mut Self {
90        self.grammars.push(grammar.into());
91        self
92    }
93
94    /// Sets the output directory. Defaults to the `OUT_DIR` environment
95    /// variable at [`generate`](Config::generate) time.
96    pub fn out_dir(&mut self, out_dir: impl Into<PathBuf>) -> &mut Self {
97        self.out_dir = Some(out_dir.into());
98        self
99    }
100
101    /// Adds a grammar library search directory (the command line's `-lib`):
102    /// imported grammars and `tokenVocab` `.tokens` files are looked up here
103    /// after the importing grammar's own directory.
104    pub fn lib_dir(&mut self, dir: impl Into<PathBuf>) -> &mut Self {
105        self.lib_dirs.push(dir.into());
106        self
107    }
108
109    /// Toggles listener (+ base listener) generation. Default: `true`.
110    pub const fn listener(&mut self, generate: bool) -> &mut Self {
111        self.gen_listener = generate;
112        self
113    }
114
115    /// Toggles visitor (+ base visitor) generation. Default: `false`.
116    pub const fn visitor(&mut self, generate: bool) -> &mut Self {
117        self.gen_visitor = generate;
118        self
119    }
120
121    /// Toggles printing of `cargo:rerun-if-changed` directives.
122    /// Default: `true`.
123    pub const fn cargo_directives(&mut self, print: bool) -> &mut Self {
124        self.cargo_directives = print;
125        self
126    }
127
128    /// Stops [`generate`](Config::generate) from printing warning diagnostics
129    /// as `cargo:warning` directives (`--quiet`, `-q`). Error diagnostics
130    /// are still printed. Default: `false`.
131    ///
132    /// This setting changes only what is printed:
133    /// [`Generation::diagnostics`] still contains all the diagnostics.
134    /// Pinned-release generation prints no `cargo:warning` directives, so
135    /// this setting has no effect on it.
136    pub const fn quiet(&mut self, enabled: bool) -> &mut Self {
137        self.quiet = enabled;
138        self
139    }
140
141    /// Precomputes static lexer DFA tables (`--static-lexer-dfa`, lexer scope
142    /// only). Default: `false`.
143    /// When the flag is off, the generated output is byte-identical to a run
144    /// without it.
145    pub const fn static_lexer_dfa(&mut self, enabled: bool) -> &mut Self {
146        self.optimization.static_lexer_dfa(enabled);
147        self
148    }
149
150    /// Precomputes static parser DFA tables (`--static-parser-dfa`, parser
151    /// scope only). Default: `false`.
152    pub const fn static_parser_dfa(&mut self, enabled: bool) -> &mut Self {
153        self.optimization.static_parser_dfa(enabled);
154        self
155    }
156
157    /// Generates only fully static lexers and parsers (`--static-dfa-only`),
158    /// which never fall back to the adaptive ATN simulation. Default: `false`.
159    /// See [`OptimizationOptions::static_dfa_only`].
160    pub const fn static_dfa_only(&mut self, enabled: bool) -> &mut Self {
161        self.optimization.static_dfa_only(enabled);
162        self
163    }
164
165    /// Renders the static parser DFA tables matching `spec` into
166    /// [`Generation::reports`] (`--dump-dfas <SPEC>`, printed to stdout by the
167    /// CLI). Implies report-only mode.
168    pub fn dump_dfas(&mut self, spec: impl Into<String>) -> &mut Self {
169        self.optimization.dump_dfas(spec);
170        self
171    }
172
173    /// Renders the time and the work that the static parser DFA pass spends
174    /// on each parser decision into [`Generation::reports`]
175    /// (`--profile-decisions`, printed to stdout by the CLI). Implies
176    /// report-only mode.
177    pub const fn profile_decisions(&mut self, enabled: bool) -> &mut Self {
178        self.optimization.profile_decisions(enabled);
179        self
180    }
181
182    /// Returns the optimization selection for in-place configuration.
183    /// See [`OptimizationOptions`] for the available options. The
184    /// convenience setters on [`Config`] delegate to it.
185    pub const fn optimization(&mut self) -> &mut OptimizationOptions {
186        &mut self.optimization
187    }
188
189    /// Marks `name` as an explicit parser entry rule (the command line's
190    /// `--entry-rule`). Repeatable.
191    pub fn entry_rule(&mut self, name: impl Into<String>) -> &mut Self {
192        self.optimization.entry_rule(name);
193        self
194    }
195
196    /// Deletes parser rules that are unreachable from the entry rules (the
197    /// command line's `--prune-unreachable-rules`). Default: `false`.
198    pub const fn prune_unreachable_rules(&mut self, enabled: bool) -> &mut Self {
199        self.optimization.prune_unreachable_rules(enabled);
200        self
201    }
202
203    /// Inlines trivial pure parser rules into their call sites (the command
204    /// line's `--inline-trivial-rules`). Default: `false`.
205    pub const fn inline_trivial_rules(&mut self, enabled: bool) -> &mut Self {
206        self.optimization.inline_trivial_rules(enabled);
207        self
208    }
209
210    /// Collapses precedence-ladder rule chains into left-recursive rules
211    /// (`--optimize-precedence-ladders`). Default: `false`.
212    pub const fn optimize_precedence_ladders(&mut self, enabled: bool) -> &mut Self {
213        self.optimization.optimize_precedence_ladders(enabled);
214        self
215    }
216
217    /// Compiles, analyzes, and optimizes the input grammars without writing
218    /// output files (`--report-only`, `--dry-run`, `-n`). Default: `false`.
219    pub const fn report_only(&mut self, enabled: bool) -> &mut Self {
220        self.optimization.report_only(enabled);
221        self
222    }
223
224    /// Reports an implicit token definition in a parser grammar (`G4S030`)
225    /// as an error instead of a warning (`--strict`). Default: `false`.
226    /// See [`OptimizationOptions::strict`].
227    pub const fn strict(&mut self, enabled: bool) -> &mut Self {
228        self.optimization.strict(enabled);
229        self
230    }
231
232    /// Switches generation to a pinned prebuilt `dbt-antlr-codegen` release
233    /// binary (requires the `download` feature).
234    ///
235    /// Instead of compiling and running the generator in-process, the
236    /// release archive for the build host's target triple is downloaded from
237    /// the GitHub release `dbt-antlr-codegen-v{version}`, verified against a
238    /// SHA-256 checksum, cached under the output directory, and invoked with
239    /// the configured arguments. Use this in build environments where the
240    /// generator's dependency tree is too expensive to compile:
241    ///
242    /// ```toml
243    /// [build-dependencies]
244    /// dbt-antlr-codegen = { version = "0.1", default-features = false, features = ["download"] }
245    /// ```
246    ///
247    /// The pinned `version` is independent of this crate's own version.
248    /// Requires `curl` on `PATH`.
249    #[cfg(feature = "download")]
250    pub fn pinned_release(&mut self, version: impl Into<String>) -> &mut Self {
251        self.pinned_release_version = Some(version.into());
252        self
253    }
254
255    /// Pins the expected SHA-256 checksum of the release archive (requires
256    /// the `download` feature and [`Config::pinned_release`]). When unset,
257    /// the checksum is read from the `.sha256` file published next to the
258    /// archive.
259    #[cfg(feature = "download")]
260    pub fn pinned_release_sha256(&mut self, sha256: impl Into<String>) -> &mut Self {
261        self.pinned_release_sha256 = Some(sha256.into());
262        self
263    }
264
265    /// Overrides the release download base URL (requires the `download`
266    /// feature), for mirrors or GitHub Enterprise hosts. Defaults to the
267    /// `sdf-labs/dbt-antlr` GitHub releases.
268    #[cfg(feature = "download")]
269    pub fn pinned_release_base_url(&mut self, base_url: impl Into<String>) -> &mut Self {
270        self.pinned_release_base_url = Some(base_url.into());
271        self
272    }
273
274    /// Generates all configured grammars into the output directory, creating
275    /// it when needed, and returns the paths written.
276    ///
277    /// When [`cargo_directives`](Config::cargo_directives) is on, each
278    /// diagnostic of the run is also printed as a `cargo:warning` directive.
279    /// With [`quiet`](Config::quiet), only the error diagnostics are printed.
280    /// Use [`generate_detailed`](Config::generate_detailed) to get the
281    /// diagnostics instead.
282    ///
283    /// # Errors
284    ///
285    /// Returns [`EmitError::Config`] when no grammars are configured or no
286    /// output directory is available, [`EmitError::Compile`] or
287    /// [`EmitError::Template`] when a grammar fails to generate, and
288    /// [`EmitError::Write`] when an output file cannot be written.
289    // Cargo's build-script protocol reads directives from stdout.
290    #[allow(clippy::print_stdout)]
291    pub fn generate(&self) -> Result<Vec<PathBuf>, EmitError> {
292        let generation = self.generate_detailed()?;
293        if self.cargo_directives {
294            for directive in cargo_warnings(generation.diagnostics(), self.quiet) {
295                println!("{directive}");
296            }
297        }
298        Ok(generation.into_written_files())
299    }
300
301    /// [`generate`](Config::generate) with the rest of the run output:
302    /// compiler diagnostics (for example unreachable parser rules), transform
303    /// notes (for example `pruned: ...` lines), and the reports of the report
304    /// modes (`dump_dfas`, `profile_decisions`).
305    ///
306    /// # Errors
307    ///
308    /// See [`generate`](Config::generate).
309    // Cargo's build-script protocol reads directives from stdout.
310    #[allow(clippy::print_stdout)]
311    pub fn generate_detailed(&self) -> Result<Generation, EmitError> {
312        if self.grammars.is_empty() {
313            return Err(EmitError::Config("no grammars configured".to_owned()));
314        }
315        if self.cargo_directives {
316            for grammar in &self.grammars {
317                println!("cargo:rerun-if-changed={}", grammar.display());
318                if let Some(parent) = grammar.parent().filter(|dir| !dir.as_os_str().is_empty()) {
319                    println!("cargo:rerun-if-changed={}", parent.display());
320                }
321            }
322            for dir in &self.lib_dirs {
323                println!("cargo:rerun-if-changed={}", dir.display());
324            }
325        }
326        if let Some(version) = &self.pinned_release_version {
327            let out_dir = self.output_dir()?;
328            create_out_dir(&out_dir)?;
329            return self.generate_pinned(version, &out_dir);
330        }
331        // Report-only runs render their reports without touching the output
332        // directory, so it stays optional here.
333        let out_dir = if self.optimization.is_report_only() {
334            None
335        } else {
336            let out_dir = self.output_dir()?;
337            create_out_dir(&out_dir)?;
338            Some(out_dir)
339        };
340        self.generate_in_process(out_dir.as_deref())
341    }
342
343    /// The configured output directory, defaulting to Cargo's `OUT_DIR`.
344    fn output_dir(&self) -> Result<PathBuf, EmitError> {
345        if let Some(dir) = &self.out_dir {
346            return Ok(dir.clone());
347        }
348        let out_dir = std::env::var_os("OUT_DIR").ok_or_else(|| {
349            EmitError::Config(
350                "no output directory: call Config::out_dir or run from a build \
351                 script (OUT_DIR is not set)"
352                    .to_owned(),
353            )
354        })?;
355        Ok(PathBuf::from(out_dir))
356    }
357
358    #[cfg(feature = "download")]
359    fn generate_pinned(
360        &self,
361        version: &str,
362        out_dir: &std::path::Path,
363    ) -> Result<Generation, EmitError> {
364        if self.optimization.has_parser_reports() {
365            return Err(EmitError::Config(
366                "dump_dfas and profile_decisions need the in-process generator; \
367                 pinned-release generation cannot render their reports"
368                    .to_owned(),
369            ));
370        }
371        let release = crate::download::PinnedRelease {
372            version: version.to_owned(),
373            expected_sha256: self.pinned_release_sha256.clone(),
374            base_url: self.pinned_release_base_url.clone(),
375        };
376        let written = crate::download::generate_with_pinned_binary(
377            &release,
378            &self.grammars,
379            &self.lib_dirs,
380            self.gen_listener,
381            self.gen_visitor,
382            &self.optimization,
383            out_dir,
384        )?;
385        Ok(Generation {
386            written,
387            diagnostics: Vec::new(),
388            optimization_messages: Vec::new(),
389            reports: Vec::new(),
390        })
391    }
392
393    #[cfg(not(feature = "download"))]
394    fn generate_pinned(
395        &self,
396        version: &str,
397        out_dir: &std::path::Path,
398    ) -> Result<Generation, EmitError> {
399        let _ = (version, out_dir);
400        let _ = (&self.pinned_release_sha256, &self.pinned_release_base_url);
401        Err(EmitError::Config(
402            "pinned_release requires the `download` feature of dbt-antlr-codegen".to_owned(),
403        ))
404    }
405
406    /// Generates in process. `out_dir` is `None` in report-only mode, where
407    /// the run renders reports instead of writing files.
408    #[cfg(feature = "generator")]
409    fn generate_in_process(
410        &self,
411        out_dir: Option<&std::path::Path>,
412    ) -> Result<Generation, EmitError> {
413        let mut written = Vec::new();
414        let mut diagnostics = Vec::new();
415        let mut optimization_messages = Vec::new();
416        let mut reports = Vec::new();
417        for grammar in &self.grammars {
418            let output = emit_files_with_options(
419                grammar,
420                &self.lib_dirs,
421                self.gen_listener,
422                self.gen_visitor,
423                &self.optimization,
424            )?;
425            if let Some(out_dir) = out_dir {
426                for file in &output.files {
427                    let path = out_dir.join(&file.name);
428                    std::fs::write(&path, &file.content).map_err(|source| EmitError::Write {
429                        path: path.clone(),
430                        source,
431                    })?;
432                    written.push(path);
433                }
434                write_optimization_manifest(
435                    out_dir,
436                    grammar,
437                    output.optimization_manifest,
438                    &mut written,
439                )?;
440            }
441            diagnostics.extend(output.diagnostics);
442            optimization_messages.extend(output.optimization_messages);
443            reports.extend(output.reports);
444        }
445        Ok(Generation {
446            written,
447            diagnostics,
448            optimization_messages,
449            reports,
450        })
451    }
452
453    #[cfg(not(feature = "generator"))]
454    fn generate_in_process(
455        &self,
456        out_dir: Option<&std::path::Path>,
457    ) -> Result<Generation, EmitError> {
458        let _ = out_dir;
459        Err(EmitError::Config(
460            "this build of dbt-antlr-codegen was compiled without the `generator` feature; \
461             enable it or use Config::pinned_release"
462                .to_owned(),
463        ))
464    }
465}
466
467/// The `cargo:warning` directives that show `diagnostics` in the cargo build
468/// output. A directive holds one line, so each line of a diagnostic gets its
469/// own directive. When `quiet` is set, only the errors get directives.
470fn cargo_warnings(diagnostics: &[Diagnostic], quiet: bool) -> Vec<String> {
471    diagnostics
472        .iter()
473        .filter(|diagnostic| !quiet || diagnostic.severity() == Severity::Error)
474        .flat_map(|diagnostic| {
475            diagnostic
476                .to_string()
477                .lines()
478                .map(|line| format!("cargo:warning={line}"))
479                .collect::<Vec<_>>()
480        })
481        .collect()
482}
483
484fn create_out_dir(out_dir: &std::path::Path) -> Result<(), EmitError> {
485    std::fs::create_dir_all(out_dir).map_err(|source| EmitError::Write {
486        path: out_dir.to_path_buf(),
487        source,
488    })
489}
490
491/// Writes the optimization manifest of `grammar` next to the generated files
492/// when the selected transforms produced one, and removes a stale manifest
493/// from an earlier optimized run when they did not.
494///
495/// The manifest file name starts with the lowercase grammar file stem
496/// (`clickhouselexer.optimizations.txt`), so that the grammars of a split
497/// grammar or of one [`Config`] do not overwrite the manifests of the others.
498/// The shared `optimizations.txt` and `optimizations.json` manifests of
499/// earlier releases are removed.
500#[cfg(feature = "generator")]
501fn write_optimization_manifest(
502    out_dir: &std::path::Path,
503    grammar: &std::path::Path,
504    manifest: Option<String>,
505    written: &mut Vec<PathBuf>,
506) -> Result<(), EmitError> {
507    let path = out_dir.join(optimization_manifest_name(grammar));
508    match manifest {
509        Some(manifest) => {
510            std::fs::write(&path, manifest).map_err(|source| EmitError::Write {
511                path: path.clone(),
512                source,
513            })?;
514            written.push(path);
515        }
516        None => remove_stale_manifest(&path)?,
517    }
518    for legacy in ["optimizations.txt", "optimizations.json"] {
519        remove_stale_manifest(&out_dir.join(legacy))?;
520    }
521    Ok(())
522}
523
524/// The file name of the optimization manifest of `grammar`, for example
525/// `clickhouseparser.optimizations.txt` for `ClickHouseParser.g4`.
526#[cfg(feature = "generator")]
527fn optimization_manifest_name(grammar: &std::path::Path) -> String {
528    let stem = grammar
529        .file_stem()
530        .map_or_else(String::new, |stem| stem.to_string_lossy().to_lowercase());
531    format!("{stem}.optimizations.txt")
532}
533
534#[cfg(feature = "generator")]
535fn remove_stale_manifest(path: &std::path::Path) -> Result<(), EmitError> {
536    match std::fs::remove_file(path) {
537        Ok(()) => Ok(()),
538        Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(()),
539        Err(source) => Err(EmitError::Write {
540            path: path.to_path_buf(),
541            source,
542        }),
543    }
544}
545
546/// Files and non-fatal output of [`Config::generate_detailed`].
547#[derive(Clone, Debug)]
548pub struct Generation {
549    written: Vec<PathBuf>,
550    diagnostics: Vec<Diagnostic>,
551    optimization_messages: Vec<String>,
552    reports: Vec<String>,
553}
554
555impl Generation {
556    /// Every file written into the output directory, in write order.
557    pub fn written_files(&self) -> &[PathBuf] {
558        &self.written
559    }
560
561    /// Diagnostics of the run that did not abort it: warnings (for example
562    /// unreachable parser rule warnings) and errors a caller must surface
563    /// (for example an unmatched `--dump-dfas` selector). Empty for
564    /// pinned-release generation, where diagnostics stay on the release
565    /// binary's stderr.
566    pub fn diagnostics(&self) -> &[Diagnostic] {
567        &self.diagnostics
568    }
569
570    /// Transform notes of the run (for example `pruned: ...` lines).
571    pub fn optimization_messages(&self) -> &[String] {
572        &self.optimization_messages
573    }
574
575    /// Rendered reports of the report modes (`dump_dfas` and
576    /// `profile_decisions`), in selection order.
577    pub fn reports(&self) -> &[String] {
578        &self.reports
579    }
580
581    /// Consumes the generation and returns the written files.
582    pub fn into_written_files(self) -> Vec<PathBuf> {
583        self.written
584    }
585}
586
587#[cfg(all(test, feature = "generator"))]
588mod tests {
589    use std::path::PathBuf;
590
591    use super::cargo_warnings;
592    use crate::error::{Diagnostic, Severity};
593
594    #[test]
595    fn cargo_warnings_give_each_diagnostic_line_a_directive() {
596        let diagnostics = [
597            Diagnostic::new(
598                "G4S030",
599                Severity::Warning,
600                "implicit definition of token CACHE in parser".to_owned(),
601                PathBuf::from("P.g4"),
602                Some((3, 24)),
603                None,
604            ),
605            Diagnostic::new(
606                "G4S192",
607                Severity::Warning,
608                "first line\nsecond line".to_owned(),
609                PathBuf::from("P.g4"),
610                None,
611                None,
612            ),
613        ];
614        assert_eq!(
615            cargo_warnings(&diagnostics, false),
616            [
617                "cargo:warning=warning[G4S030]: P.g4:3:24: implicit definition of token CACHE in parser",
618                "cargo:warning=warning[G4S192]: P.g4: first line",
619                "cargo:warning=second line",
620            ]
621        );
622    }
623
624    #[test]
625    fn quiet_cargo_warnings_keep_only_the_errors() {
626        let diagnostics = [
627            Diagnostic::new(
628                "G4S192",
629                Severity::Warning,
630                "lookahead budget".to_owned(),
631                PathBuf::from("P.g4"),
632                Some((7, 1)),
633                None,
634            ),
635            Diagnostic::new(
636                "G4S030",
637                Severity::Error,
638                "implicit definition of token CACHE in parser".to_owned(),
639                PathBuf::from("P.g4"),
640                Some((3, 24)),
641                None,
642            ),
643        ];
644        assert_eq!(
645            cargo_warnings(&diagnostics, true),
646            [
647                "cargo:warning=error[G4S030]: P.g4:3:24: implicit definition of token CACHE in parser"
648            ]
649        );
650    }
651}