use std::path::{Path, PathBuf};
use dbt_antlr_runtime::analysis::EOF;
use crate::grammar::compiler::compile;
use crate::grammar::loader::LoadOptions;
use crate::optimization::AtnFallback;
use crate::optimization::parser_dfa::cost::{AttemptKind, AttemptOutcome};
use crate::optimization::parser_dfa::table::{StaticDfaState, StaticDfaTable};
use super::{Category, DecisionClassifier, DecisionResult};
fn grammars_dir() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("crate directory")
.join("dbt-antlr-runtime")
.join("grammars")
}
fn fixture(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests")
.join("codegen-direct")
.join("fixtures")
.join(name)
}
fn classifier(parser: &crate::grammar::atn::CompiledParser) -> DecisionClassifier {
let ll1 = crate::optimization::static_parser_dfa::ll1_disjointness(parser);
DecisionClassifier::new(parser.atn, ll1).with_descent_rules(
crate::optimization::static_parser_dfa::descent_rules(parser),
)
}
fn classify(path: &Path) -> Vec<DecisionResult> {
classify_with(path, AtnFallback::Allowed)
}
fn classify_with(path: &Path, atn_fallback: AtnFallback) -> Vec<DecisionResult> {
let compilation = compile(LoadOptions {
roots: vec![path.to_path_buf()],
library_directories: Vec::new(),
})
.expect("grammar compiles");
let mut results = Vec::new();
for root in &compilation.roots {
let Some(parser_id) = root.parser else {
continue;
};
let parser = compilation.parser(parser_id).expect("parser artifact");
results.extend(
classifier(parser)
.with_atn_fallback(atn_fallback)
.classify_all(),
);
}
results
}
fn rule_names(path: &Path, results: &[DecisionResult]) -> Vec<String> {
let compilation = compile(LoadOptions {
roots: vec![path.to_path_buf()],
library_directories: Vec::new(),
})
.expect("grammar compiles");
let Some(root) = compilation.roots.first() else {
return Vec::new();
};
let Some(parser) = compilation.parser(root.parser.expect("parser root")) else {
return Vec::new();
};
results
.iter()
.map(|result| parser.semantic.recognizer.rule_names[result.rule_index].clone())
.collect()
}
fn categories(path: &Path) -> Vec<Category> {
classify(path)
.into_iter()
.map(|result| result.category)
.collect()
}
#[test]
fn golden_corpus_categories_match_the_reference_classifier() {
let cases: [(&str, &[Category]); 7] = [
("CSV.g4", &[Category::LL1; 4]),
(
"Labels.g4",
&[
Category::LL1,
Category::LL1,
Category::LLK,
Category::LR_PRECEDENCE,
],
),
("SimpleLR.g4", &[Category::LR_PRECEDENCE]),
("VisitorBasic.g4", &[]),
("VisitorCalc.g4", &[Category::LLK, Category::LR_PRECEDENCE]),
("ReferenceToATN.g4", &[Category::HYBRID, Category::LL1]),
(
"Perf.g4",
&[
Category::EXACT_AMBIG,
Category::LL1,
Category::LLK,
Category::LR_PRECEDENCE,
],
),
];
for (file, expected) in cases {
let path = grammars_dir().join(file);
assert_eq!(categories(&path), expected, "{file}");
}
let shapes = fixture("parser-shapes").join("ParserShapes.g4");
assert_eq!(
categories(&shapes),
[
Category::LL1,
Category::HYBRID,
Category::LL1,
Category::NON_GREEDY,
]
);
let results = classify(&shapes);
let rules = rule_names(&shapes, &results);
assert_eq!(rules, ["optional", "plus", "star", "nongreedy"]);
}
#[test]
fn exhausted_visit_budget_yields_overflow_without_a_table() {
let reference = classify_with_budget(&grammars_dir().join("ReferenceToATN.g4"), 1);
let decision = &reference[0];
assert_eq!(decision.category, Category::OVERFLOW);
assert!(decision.table.is_none());
assert!(!decision.is_fully_static());
let precedence = classify_with_budget(&grammars_dir().join("SimpleLR.g4"), 1);
let decision = &precedence[0];
assert_eq!(decision.category, Category::LR_PRECEDENCE);
let classes = decision.precedence.as_ref().expect("precedence classes");
assert!(classes.adaptive_classes > 0);
assert!(classes.tables.iter().all(Option::is_none));
assert!(!decision.is_fully_static());
}
fn classify_with_budget(path: &Path, budget: u64) -> Vec<DecisionResult> {
let compilation = compile(LoadOptions {
roots: vec![path.to_path_buf()],
library_directories: Vec::new(),
})
.expect("grammar compiles");
let root = compilation.roots.first().expect("one root");
let parser = compilation
.parser(root.parser.expect("parser root"))
.expect("parser artifact");
classifier(parser).with_visit_budget(budget).classify_all()
}
fn token_types(path: &Path, names: &[&str]) -> Vec<i32> {
let compilation = compile(LoadOptions {
roots: vec![path.to_path_buf()],
library_directories: Vec::new(),
})
.expect("grammar compiles");
let root = compilation.roots.first().expect("one root");
let parser = compilation
.parser(root.parser.expect("parser root"))
.expect("parser artifact");
let recognizer = &parser.semantic.recognizer;
names
.iter()
.map(|name| {
let position = |names: &[Option<String>]| {
names
.iter()
.position(|candidate| candidate.as_deref() == Some(*name))
};
position(&recognizer.literal_names)
.or_else(|| position(&recognizer.symbolic_names))
.unwrap_or_else(|| panic!("unknown token {name}")) as i32
})
.collect()
}
fn walk<'t>(table: &'t StaticDfaTable, tokens: &[i32]) -> Option<&'t StaticDfaState> {
let mut state = &table.states[0];
for token in tokens {
let edge = state
.edges
.iter()
.find(|edge| edge.lo <= *token && *token <= edge.hi)?;
state = &table.states[edge.target];
}
Some(state)
}
fn attempts_of(results: &[DecisionResult], decision: usize) -> Vec<AttemptKind> {
results[decision]
.cost
.attempts
.iter()
.map(|attempt| attempt.kind)
.collect()
}
#[test]
fn witness_pass_repeats_only_the_stopped_attempt() {
let path = fixture("vscode-split").join("TParser.g4");
let results = classify_with_budget(&path, super::DEFAULT_VISIT_BUDGET);
assert_eq!(results[5].category, Category::PREDICATED);
assert!(results[5].table.is_none());
assert_eq!(
attempts_of(&results, 5),
[
AttemptKind::Unbounded { abort: true },
AttemptKind::Hybrid,
AttemptKind::Unbounded { abort: false }
]
);
}
#[test]
fn unstopped_first_pass_needs_no_witness_pass() {
let path = grammars_dir().join("parser").join("LowCoverage.g4");
let results = classify_with_budget(&path, super::DEFAULT_VISIT_BUDGET);
assert_eq!(results[0].category, Category::PREDICATED);
assert!(results[0].table.is_none());
assert_eq!(
attempts_of(&results, 0),
[AttemptKind::Unbounded { abort: true }, AttemptKind::Hybrid]
);
}
#[test]
fn exhausted_start_state_skips_the_hybrid_attempts() {
let path = grammars_dir().join("ReferenceToATN.g4");
let results = classify_with_budget(&path, 1);
assert_eq!(results[0].category, Category::OVERFLOW);
assert_eq!(
attempts_of(&results, 0),
[
AttemptKind::Unbounded { abort: true },
AttemptKind::Widened { abort: true }
]
);
}
#[test]
fn stopped_unbounded_attempt_does_not_fall_back_to_widening() {
let results = classify_source(
"WidenedAfterStop",
"grammar WidenedAfterStop;\n\
top : a X | a Y ;\n\
a : ( A | C ( B A | B a E ) ) D ;\n\
A : 'a' ; B : 'b' ; C : 'c' ; D : 'd' ; E : 'e' ; X : 'x' ; Y : 'y' ;\n",
);
let decision = &results[1];
assert_eq!(decision.category, Category::HYBRID);
assert_eq!(decision.conflicts, [vec![1, 2]]);
assert!(decision.table.is_some());
}
#[test]
fn widened_competitors_get_no_uniform_trust() {
let path = grammars_dir().join("parser").join("WidenedTrust.g4");
let results = classify(&path);
let decision = &results[1];
assert_eq!(
attempts_of(&results, 1),
[
AttemptKind::Unbounded { abort: true },
AttemptKind::Hybrid,
AttemptKind::Unbounded { abort: false },
AttemptKind::Widened { abort: false },
]
);
assert_eq!(
decision.cost.attempts[2].outcome,
AttemptOutcome::StateBudget
);
assert_eq!(decision.category, Category::CONTEXT_SENSITIVE);
assert_eq!(decision.utrust_resolutions, 0);
assert!(decision.table.is_none());
assert!(!decision.is_fully_static());
let static_only = classify_with(&path, AtnFallback::Forbidden);
let decision = &static_only[1];
assert_eq!(decision.category, Category::HYBRID);
let types = token_types(&path, &["'a'", "'b'", "'d'", "'e'"]);
let (a, b, d, e) = (types[0], types[1], types[2], types[3]);
let table = decision.table.as_ref().expect("static-only table");
let conflict = walk(table, &[b, b, b, b, a, d, e]).expect("conflict state");
assert!(conflict.is_escape(), "the conflict is a reported escape");
assert!(
walk(table, &[b, b, b, b, a]).is_some_and(|state| state.accept == 0),
"the table does not decide after `'b' 'b' 'b' 'b' 'a'`"
);
}
#[test]
fn attempt_codes_follow_the_classes_and_the_fallback_policy() {
let results = classify_with_budget(
&grammars_dir().join("parser").join("LoopFrontier.g4"),
2_000,
);
assert_eq!(
results[4].cost.attempts_text(),
"c0: U! H$; c1: U! H$; c2: U! H$; c3: U; c4: U"
);
let path = grammars_dir().join("ReferenceToATN.g4");
assert_eq!(classify(&path)[0].cost.attempts_text(), "U! H");
assert_eq!(
classify_with(&path, AtnFallback::Forbidden)[0]
.cost
.attempts_text(),
"U! S"
);
}
#[test]
fn report_decisions_prints_the_decision_inventory() {
let output = emit_with(|options| {
options.report_decisions(true);
});
assert!(output.files.is_empty(), "report mode emits no files");
let manifest = output
.optimization_manifest
.expect("manifest is still rendered");
assert!(manifest.contains("\"reportOnly\": True"), "{manifest}");
let [report] = output.reports.as_slice() else {
panic!("one report per parser unit: {:?}", output.reports);
};
assert_eq!(
report.lines().next(),
Some("=== parser decision report: CSVParser (4 decisions) ==="),
"{report}"
);
assert!(report.contains("summary: LL1=4"), "{report}");
for row in [
" 0 csvFile 12 LL1 1 0 0",
" 1 row 21 LL1 1 0 0",
" 2 row 25 LL1 1 0 0",
" 3 field 32 LL1 1 0 0",
] {
assert!(report.contains(row), "missing {row:?} in:\n{report}");
}
}