dbt_antlr_codegen/config.rs
1// SPDX-License-Identifier: BSD-3-Clause
2// Copyright (c) 2026 Bo Lin
3//! Build-script entry point: generate recognizers during `cargo build`, in
4//! the style of `prost-build` or `lalrpop`.
5
6use std::path::PathBuf;
7
8use crate::OptimizationOptions;
9#[cfg(feature = "generator")]
10use crate::emission::emit_files_with_options;
11use crate::error::{Diagnostic, EmitError, Severity};
12
13/// Generator configuration, primarily for use from a `build.rs` build script.
14///
15/// With the generator as a build dependency, no separate tool installation is
16/// needed and the generator version is pinned by `Cargo.lock` like any other
17/// dependency:
18///
19/// ```toml
20/// [dependencies]
21/// dbt-antlr-runtime = "0.1"
22///
23/// [build-dependencies]
24/// # default-features = false skips the `fancy` diagnostics rendering the
25/// # command-line tool uses; it shrinks the build-dependency tree by ~40%.
26/// dbt-antlr-codegen = { version = "0.1", default-features = false, features = ["generator"] }
27/// ```
28///
29/// ```rust,no_run
30/// // build.rs
31/// dbt_antlr_codegen::Config::new("grammars/Expr.g4")
32/// .generate()
33/// .unwrap_or_else(|error| panic!("{error:?}"));
34/// ```
35///
36/// Generated files land in the [output directory](Config::out_dir), which
37/// defaults to the `OUT_DIR` environment variable cargo sets for build
38/// scripts; pull them into the crate with `include!`:
39///
40/// ```rust,ignore
41/// mod expr {
42/// #![allow(clippy::all)]
43/// include!(concat!(env!("OUT_DIR"), "/exprparser.rs"));
44/// }
45/// ```
46///
47/// [`generate`](Config::generate) prints `cargo:rerun-if-changed` directives
48/// for the configured grammars and library directories (including each root
49/// grammar's own directory, which the loader searches for imported grammars),
50/// so the build script reruns exactly when grammar inputs change. It also
51/// prints each compiler diagnostic as a `cargo:warning` directive, so that
52/// cargo shows the warnings of the grammar (for example `G4S030` for a token
53/// that the lexer never emits) in the build output. Call
54/// [`quiet`](Config::quiet) to print only the errors.
55#[derive(Clone, Debug)]
56pub struct Config {
57 grammars: Vec<PathBuf>,
58 out_dir: Option<PathBuf>,
59 lib_dirs: Vec<PathBuf>,
60 gen_listener: bool,
61 gen_visitor: bool,
62 cargo_directives: bool,
63 quiet: bool,
64 optimization: OptimizationOptions,
65 pinned_release_version: Option<String>,
66 pinned_release_sha256: Option<String>,
67 pinned_release_base_url: Option<String>,
68}
69
70impl Config {
71 /// Starts a configuration generating from `grammar`.
72 pub fn new(grammar: impl Into<PathBuf>) -> Self {
73 Self {
74 grammars: vec![grammar.into()],
75 out_dir: None,
76 lib_dirs: Vec::new(),
77 gen_listener: true,
78 gen_visitor: false,
79 cargo_directives: true,
80 quiet: false,
81 optimization: OptimizationOptions::default(),
82 pinned_release_version: None,
83 pinned_release_sha256: None,
84 pinned_release_base_url: None,
85 }
86 }
87
88 /// Adds another root grammar to generate from.
89 pub fn grammar(&mut self, grammar: impl Into<PathBuf>) -> &mut Self {
90 self.grammars.push(grammar.into());
91 self
92 }
93
94 /// Sets the output directory. Defaults to the `OUT_DIR` environment
95 /// variable at [`generate`](Config::generate) time.
96 pub fn out_dir(&mut self, out_dir: impl Into<PathBuf>) -> &mut Self {
97 self.out_dir = Some(out_dir.into());
98 self
99 }
100
101 /// Adds a grammar library search directory (the command line's `-lib`):
102 /// imported grammars and `tokenVocab` `.tokens` files are looked up here
103 /// after the importing grammar's own directory.
104 pub fn lib_dir(&mut self, dir: impl Into<PathBuf>) -> &mut Self {
105 self.lib_dirs.push(dir.into());
106 self
107 }
108
109 /// Toggles listener (+ base listener) generation. Default: `true`.
110 pub const fn listener(&mut self, generate: bool) -> &mut Self {
111 self.gen_listener = generate;
112 self
113 }
114
115 /// Toggles visitor (+ base visitor) generation. Default: `false`.
116 pub const fn visitor(&mut self, generate: bool) -> &mut Self {
117 self.gen_visitor = generate;
118 self
119 }
120
121 /// Toggles printing of `cargo:rerun-if-changed` directives.
122 /// Default: `true`.
123 pub const fn cargo_directives(&mut self, print: bool) -> &mut Self {
124 self.cargo_directives = print;
125 self
126 }
127
128 /// Stops [`generate`](Config::generate) from printing warning diagnostics
129 /// as `cargo:warning` directives (`--quiet`, `-q`). Error diagnostics
130 /// are still printed. Default: `false`.
131 ///
132 /// This setting changes only what is printed:
133 /// [`Generation::diagnostics`] still contains all the diagnostics.
134 /// Pinned-release generation prints no `cargo:warning` directives, so
135 /// this setting has no effect on it.
136 pub const fn quiet(&mut self, enabled: bool) -> &mut Self {
137 self.quiet = enabled;
138 self
139 }
140
141 /// Precomputes static lexer DFA tables (`--static-lexer-dfa`, lexer scope
142 /// only). Default: `false`.
143 /// When the flag is off, the generated output is byte-identical to a run
144 /// without it.
145 pub const fn static_lexer_dfa(&mut self, enabled: bool) -> &mut Self {
146 self.optimization.static_lexer_dfa(enabled);
147 self
148 }
149
150 /// Precomputes static parser DFA tables (`--static-parser-dfa`, parser
151 /// scope only). Default: `false`.
152 pub const fn static_parser_dfa(&mut self, enabled: bool) -> &mut Self {
153 self.optimization.static_parser_dfa(enabled);
154 self
155 }
156
157 /// Generates only fully static lexers and parsers (`--static-dfa-only`),
158 /// which never fall back to the adaptive ATN simulation. Default: `false`.
159 /// See [`OptimizationOptions::static_dfa_only`].
160 pub const fn static_dfa_only(&mut self, enabled: bool) -> &mut Self {
161 self.optimization.static_dfa_only(enabled);
162 self
163 }
164
165 /// Renders the static parser DFA tables matching `spec` into
166 /// [`Generation::reports`] (`--dump-dfas <SPEC>`, printed to stdout by the
167 /// CLI). Implies report-only mode.
168 pub fn dump_dfas(&mut self, spec: impl Into<String>) -> &mut Self {
169 self.optimization.dump_dfas(spec);
170 self
171 }
172
173 /// Renders the time and the work that the static parser DFA pass spends
174 /// on each parser decision into [`Generation::reports`]
175 /// (`--profile-decisions`, printed to stdout by the CLI). Implies
176 /// report-only mode.
177 pub const fn profile_decisions(&mut self, enabled: bool) -> &mut Self {
178 self.optimization.profile_decisions(enabled);
179 self
180 }
181
182 /// Returns the optimization selection for in-place configuration.
183 /// See [`OptimizationOptions`] for the available options. The
184 /// convenience setters on [`Config`] delegate to it.
185 pub const fn optimization(&mut self) -> &mut OptimizationOptions {
186 &mut self.optimization
187 }
188
189 /// Marks `name` as an explicit parser entry rule (the command line's
190 /// `--entry-rule`). Repeatable.
191 pub fn entry_rule(&mut self, name: impl Into<String>) -> &mut Self {
192 self.optimization.entry_rule(name);
193 self
194 }
195
196 /// Deletes parser rules that are unreachable from the entry rules (the
197 /// command line's `--prune-unreachable-rules`). Default: `false`.
198 pub const fn prune_unreachable_rules(&mut self, enabled: bool) -> &mut Self {
199 self.optimization.prune_unreachable_rules(enabled);
200 self
201 }
202
203 /// Inlines trivial pure parser rules into their call sites (the command
204 /// line's `--inline-trivial-rules`). Default: `false`.
205 pub const fn inline_trivial_rules(&mut self, enabled: bool) -> &mut Self {
206 self.optimization.inline_trivial_rules(enabled);
207 self
208 }
209
210 /// Collapses precedence-ladder rule chains into left-recursive rules
211 /// (`--optimize-precedence-ladders`). Default: `false`.
212 pub const fn optimize_precedence_ladders(&mut self, enabled: bool) -> &mut Self {
213 self.optimization.optimize_precedence_ladders(enabled);
214 self
215 }
216
217 /// Compiles, analyzes, and optimizes the input grammars without writing
218 /// output files (`--report-only`, `--dry-run`, `-n`). Default: `false`.
219 pub const fn report_only(&mut self, enabled: bool) -> &mut Self {
220 self.optimization.report_only(enabled);
221 self
222 }
223
224 /// Reports an implicit token definition in a parser grammar (`G4S030`)
225 /// as an error instead of a warning (`--strict`). Default: `false`.
226 /// See [`OptimizationOptions::strict`].
227 pub const fn strict(&mut self, enabled: bool) -> &mut Self {
228 self.optimization.strict(enabled);
229 self
230 }
231
232 /// Switches generation to a pinned prebuilt `dbt-antlr-codegen` release
233 /// binary (requires the `download` feature).
234 ///
235 /// Instead of compiling and running the generator in-process, the
236 /// release archive for the build host's target triple is downloaded from
237 /// the GitHub release `dbt-antlr-codegen-v{version}`, verified against a
238 /// SHA-256 checksum, cached under the output directory, and invoked with
239 /// the configured arguments. Use this in build environments where the
240 /// generator's dependency tree is too expensive to compile:
241 ///
242 /// ```toml
243 /// [build-dependencies]
244 /// dbt-antlr-codegen = { version = "0.1", default-features = false, features = ["download"] }
245 /// ```
246 ///
247 /// The pinned `version` is independent of this crate's own version.
248 /// Requires `curl` on `PATH`.
249 #[cfg(feature = "download")]
250 pub fn pinned_release(&mut self, version: impl Into<String>) -> &mut Self {
251 self.pinned_release_version = Some(version.into());
252 self
253 }
254
255 /// Pins the expected SHA-256 checksum of the release archive (requires
256 /// the `download` feature and [`Config::pinned_release`]). When unset,
257 /// the checksum is read from the `.sha256` file published next to the
258 /// archive.
259 #[cfg(feature = "download")]
260 pub fn pinned_release_sha256(&mut self, sha256: impl Into<String>) -> &mut Self {
261 self.pinned_release_sha256 = Some(sha256.into());
262 self
263 }
264
265 /// Overrides the release download base URL (requires the `download`
266 /// feature), for mirrors or GitHub Enterprise hosts. Defaults to the
267 /// `sdf-labs/dbt-antlr` GitHub releases.
268 #[cfg(feature = "download")]
269 pub fn pinned_release_base_url(&mut self, base_url: impl Into<String>) -> &mut Self {
270 self.pinned_release_base_url = Some(base_url.into());
271 self
272 }
273
274 /// Generates all configured grammars into the output directory, creating
275 /// it when needed, and returns the paths written.
276 ///
277 /// When [`cargo_directives`](Config::cargo_directives) is on, each
278 /// diagnostic of the run is also printed as a `cargo:warning` directive.
279 /// With [`quiet`](Config::quiet), only the error diagnostics are printed.
280 /// Use [`generate_detailed`](Config::generate_detailed) to get the
281 /// diagnostics instead.
282 ///
283 /// # Errors
284 ///
285 /// Returns [`EmitError::Config`] when no grammars are configured or no
286 /// output directory is available, [`EmitError::Compile`] or
287 /// [`EmitError::Template`] when a grammar fails to generate, and
288 /// [`EmitError::Write`] when an output file cannot be written.
289 // Cargo's build-script protocol reads directives from stdout.
290 #[allow(clippy::print_stdout)]
291 pub fn generate(&self) -> Result<Vec<PathBuf>, EmitError> {
292 let generation = self.generate_detailed()?;
293 if self.cargo_directives {
294 for directive in cargo_warnings(generation.diagnostics(), self.quiet) {
295 println!("{directive}");
296 }
297 }
298 Ok(generation.into_written_files())
299 }
300
301 /// [`generate`](Config::generate) with the rest of the run output:
302 /// compiler diagnostics (for example unreachable parser rules), transform
303 /// notes (for example `pruned: ...` lines), and the reports of the report
304 /// modes (`dump_dfas`, `profile_decisions`).
305 ///
306 /// # Errors
307 ///
308 /// See [`generate`](Config::generate).
309 // Cargo's build-script protocol reads directives from stdout.
310 #[allow(clippy::print_stdout)]
311 pub fn generate_detailed(&self) -> Result<Generation, EmitError> {
312 if self.grammars.is_empty() {
313 return Err(EmitError::Config("no grammars configured".to_owned()));
314 }
315 if self.cargo_directives {
316 for grammar in &self.grammars {
317 println!("cargo:rerun-if-changed={}", grammar.display());
318 if let Some(parent) = grammar.parent().filter(|dir| !dir.as_os_str().is_empty()) {
319 println!("cargo:rerun-if-changed={}", parent.display());
320 }
321 }
322 for dir in &self.lib_dirs {
323 println!("cargo:rerun-if-changed={}", dir.display());
324 }
325 }
326 if let Some(version) = &self.pinned_release_version {
327 let out_dir = self.output_dir()?;
328 create_out_dir(&out_dir)?;
329 return self.generate_pinned(version, &out_dir);
330 }
331 // Report-only runs render their reports without touching the output
332 // directory, so it stays optional here.
333 let out_dir = if self.optimization.is_report_only() {
334 None
335 } else {
336 let out_dir = self.output_dir()?;
337 create_out_dir(&out_dir)?;
338 Some(out_dir)
339 };
340 self.generate_in_process(out_dir.as_deref())
341 }
342
343 /// The configured output directory, defaulting to Cargo's `OUT_DIR`.
344 fn output_dir(&self) -> Result<PathBuf, EmitError> {
345 if let Some(dir) = &self.out_dir {
346 return Ok(dir.clone());
347 }
348 let out_dir = std::env::var_os("OUT_DIR").ok_or_else(|| {
349 EmitError::Config(
350 "no output directory: call Config::out_dir or run from a build \
351 script (OUT_DIR is not set)"
352 .to_owned(),
353 )
354 })?;
355 Ok(PathBuf::from(out_dir))
356 }
357
358 #[cfg(feature = "download")]
359 fn generate_pinned(
360 &self,
361 version: &str,
362 out_dir: &std::path::Path,
363 ) -> Result<Generation, EmitError> {
364 if self.optimization.has_parser_reports() {
365 return Err(EmitError::Config(
366 "dump_dfas and profile_decisions need the in-process generator; \
367 pinned-release generation cannot render their reports"
368 .to_owned(),
369 ));
370 }
371 let release = crate::download::PinnedRelease {
372 version: version.to_owned(),
373 expected_sha256: self.pinned_release_sha256.clone(),
374 base_url: self.pinned_release_base_url.clone(),
375 };
376 let written = crate::download::generate_with_pinned_binary(
377 &release,
378 &self.grammars,
379 &self.lib_dirs,
380 self.gen_listener,
381 self.gen_visitor,
382 &self.optimization,
383 out_dir,
384 )?;
385 Ok(Generation {
386 written,
387 diagnostics: Vec::new(),
388 optimization_messages: Vec::new(),
389 reports: Vec::new(),
390 })
391 }
392
393 #[cfg(not(feature = "download"))]
394 fn generate_pinned(
395 &self,
396 version: &str,
397 out_dir: &std::path::Path,
398 ) -> Result<Generation, EmitError> {
399 let _ = (version, out_dir);
400 let _ = (&self.pinned_release_sha256, &self.pinned_release_base_url);
401 Err(EmitError::Config(
402 "pinned_release requires the `download` feature of dbt-antlr-codegen".to_owned(),
403 ))
404 }
405
406 /// Generates in process. `out_dir` is `None` in report-only mode, where
407 /// the run renders reports instead of writing files.
408 #[cfg(feature = "generator")]
409 fn generate_in_process(
410 &self,
411 out_dir: Option<&std::path::Path>,
412 ) -> Result<Generation, EmitError> {
413 let mut written = Vec::new();
414 let mut diagnostics = Vec::new();
415 let mut optimization_messages = Vec::new();
416 let mut reports = Vec::new();
417 for grammar in &self.grammars {
418 let output = emit_files_with_options(
419 grammar,
420 &self.lib_dirs,
421 self.gen_listener,
422 self.gen_visitor,
423 &self.optimization,
424 )?;
425 if let Some(out_dir) = out_dir {
426 for file in &output.files {
427 let path = out_dir.join(&file.name);
428 std::fs::write(&path, &file.content).map_err(|source| EmitError::Write {
429 path: path.clone(),
430 source,
431 })?;
432 written.push(path);
433 }
434 write_optimization_manifest(
435 out_dir,
436 grammar,
437 output.optimization_manifest,
438 &mut written,
439 )?;
440 }
441 diagnostics.extend(output.diagnostics);
442 optimization_messages.extend(output.optimization_messages);
443 reports.extend(output.reports);
444 }
445 Ok(Generation {
446 written,
447 diagnostics,
448 optimization_messages,
449 reports,
450 })
451 }
452
453 #[cfg(not(feature = "generator"))]
454 fn generate_in_process(
455 &self,
456 out_dir: Option<&std::path::Path>,
457 ) -> Result<Generation, EmitError> {
458 let _ = out_dir;
459 Err(EmitError::Config(
460 "this build of dbt-antlr-codegen was compiled without the `generator` feature; \
461 enable it or use Config::pinned_release"
462 .to_owned(),
463 ))
464 }
465}
466
467/// The `cargo:warning` directives that show `diagnostics` in the cargo build
468/// output. A directive holds one line, so each line of a diagnostic gets its
469/// own directive. When `quiet` is set, only the errors get directives.
470fn cargo_warnings(diagnostics: &[Diagnostic], quiet: bool) -> Vec<String> {
471 diagnostics
472 .iter()
473 .filter(|diagnostic| !quiet || diagnostic.severity() == Severity::Error)
474 .flat_map(|diagnostic| {
475 diagnostic
476 .to_string()
477 .lines()
478 .map(|line| format!("cargo:warning={line}"))
479 .collect::<Vec<_>>()
480 })
481 .collect()
482}
483
484fn create_out_dir(out_dir: &std::path::Path) -> Result<(), EmitError> {
485 std::fs::create_dir_all(out_dir).map_err(|source| EmitError::Write {
486 path: out_dir.to_path_buf(),
487 source,
488 })
489}
490
491/// Writes the optimization manifest of `grammar` next to the generated files
492/// when the selected transforms produced one, and removes a stale manifest
493/// from an earlier optimized run when they did not.
494///
495/// The manifest file name starts with the lowercase grammar file stem
496/// (`clickhouselexer.optimizations.txt`), so that the grammars of a split
497/// grammar or of one [`Config`] do not overwrite the manifests of the others.
498/// The shared `optimizations.txt` and `optimizations.json` manifests of
499/// earlier releases are removed.
500#[cfg(feature = "generator")]
501fn write_optimization_manifest(
502 out_dir: &std::path::Path,
503 grammar: &std::path::Path,
504 manifest: Option<String>,
505 written: &mut Vec<PathBuf>,
506) -> Result<(), EmitError> {
507 let path = out_dir.join(optimization_manifest_name(grammar));
508 match manifest {
509 Some(manifest) => {
510 std::fs::write(&path, manifest).map_err(|source| EmitError::Write {
511 path: path.clone(),
512 source,
513 })?;
514 written.push(path);
515 }
516 None => remove_stale_manifest(&path)?,
517 }
518 for legacy in ["optimizations.txt", "optimizations.json"] {
519 remove_stale_manifest(&out_dir.join(legacy))?;
520 }
521 Ok(())
522}
523
524/// The file name of the optimization manifest of `grammar`, for example
525/// `clickhouseparser.optimizations.txt` for `ClickHouseParser.g4`.
526#[cfg(feature = "generator")]
527fn optimization_manifest_name(grammar: &std::path::Path) -> String {
528 let stem = grammar
529 .file_stem()
530 .map_or_else(String::new, |stem| stem.to_string_lossy().to_lowercase());
531 format!("{stem}.optimizations.txt")
532}
533
534#[cfg(feature = "generator")]
535fn remove_stale_manifest(path: &std::path::Path) -> Result<(), EmitError> {
536 match std::fs::remove_file(path) {
537 Ok(()) => Ok(()),
538 Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(()),
539 Err(source) => Err(EmitError::Write {
540 path: path.to_path_buf(),
541 source,
542 }),
543 }
544}
545
546/// Files and non-fatal output of [`Config::generate_detailed`].
547#[derive(Clone, Debug)]
548pub struct Generation {
549 written: Vec<PathBuf>,
550 diagnostics: Vec<Diagnostic>,
551 optimization_messages: Vec<String>,
552 reports: Vec<String>,
553}
554
555impl Generation {
556 /// Every file written into the output directory, in write order.
557 pub fn written_files(&self) -> &[PathBuf] {
558 &self.written
559 }
560
561 /// Diagnostics of the run that did not abort it: warnings (for example
562 /// unreachable parser rule warnings) and errors a caller must surface
563 /// (for example an unmatched `--dump-dfas` selector). Empty for
564 /// pinned-release generation, where diagnostics stay on the release
565 /// binary's stderr.
566 pub fn diagnostics(&self) -> &[Diagnostic] {
567 &self.diagnostics
568 }
569
570 /// Transform notes of the run (for example `pruned: ...` lines).
571 pub fn optimization_messages(&self) -> &[String] {
572 &self.optimization_messages
573 }
574
575 /// Rendered reports of the report modes (`dump_dfas` and
576 /// `profile_decisions`), in selection order.
577 pub fn reports(&self) -> &[String] {
578 &self.reports
579 }
580
581 /// Consumes the generation and returns the written files.
582 pub fn into_written_files(self) -> Vec<PathBuf> {
583 self.written
584 }
585}
586
587#[cfg(all(test, feature = "generator"))]
588mod tests {
589 use std::path::PathBuf;
590
591 use super::cargo_warnings;
592 use crate::error::{Diagnostic, Severity};
593
594 #[test]
595 fn cargo_warnings_give_each_diagnostic_line_a_directive() {
596 let diagnostics = [
597 Diagnostic::new(
598 "G4S030",
599 Severity::Warning,
600 "implicit definition of token CACHE in parser".to_owned(),
601 PathBuf::from("P.g4"),
602 Some((3, 24)),
603 None,
604 ),
605 Diagnostic::new(
606 "G4S192",
607 Severity::Warning,
608 "first line\nsecond line".to_owned(),
609 PathBuf::from("P.g4"),
610 None,
611 None,
612 ),
613 ];
614 assert_eq!(
615 cargo_warnings(&diagnostics, false),
616 [
617 "cargo:warning=warning[G4S030]: P.g4:3:24: implicit definition of token CACHE in parser",
618 "cargo:warning=warning[G4S192]: P.g4: first line",
619 "cargo:warning=second line",
620 ]
621 );
622 }
623
624 #[test]
625 fn quiet_cargo_warnings_keep_only_the_errors() {
626 let diagnostics = [
627 Diagnostic::new(
628 "G4S192",
629 Severity::Warning,
630 "lookahead budget".to_owned(),
631 PathBuf::from("P.g4"),
632 Some((7, 1)),
633 None,
634 ),
635 Diagnostic::new(
636 "G4S030",
637 Severity::Error,
638 "implicit definition of token CACHE in parser".to_owned(),
639 PathBuf::from("P.g4"),
640 Some((3, 24)),
641 None,
642 ),
643 ];
644 assert_eq!(
645 cargo_warnings(&diagnostics, true),
646 [
647 "cargo:warning=error[G4S030]: P.g4:3:24: implicit definition of token CACHE in parser"
648 ]
649 );
650 }
651}