use std::collections::BTreeMap;
use regex::Regex;
use serde::Serialize;
use crate::ast::{Expr, Grammar, GrammarItem, RuleDecl, TokenBody};
use crate::error::RantlrError;
use crate::span::Span;
use crate::token::BuiltinType;
use crate::{analyze, Diagnostic};
#[derive(Debug, Clone, Serialize)]
pub struct ParseTree {
pub kind: String,
pub text: String,
pub children: Vec<ParseTree>,
}
#[derive(Debug, Clone, Serialize)]
pub struct AnalyzeResult {
pub grammar_name: String,
pub tree: ParseTree,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DiagnosticJson {
pub severity: String,
pub kind: String,
pub line: usize,
pub column: usize,
pub end_line: usize,
pub end_column: usize,
pub message: String,
pub help: Option<String>,
pub code: Option<String>,
pub fix_hint: Option<crate::FixHint>,
}
pub fn diagnose(source: &str) -> Vec<DiagnosticJson> {
match crate::parser::parse(source) {
Err(err) => vec![diagnostic_from_error(source, &err)],
Ok(grammar) => {
let mut out = Vec::new();
for diag in crate::linter::lint_all(source, &grammar) {
out.push(diagnostic_from_diag(source, &diag));
}
out
}
}
}
pub fn analyze_and_parse(source: &str, test_input: &str) -> Result<AnalyzeResult, RantlrError> {
let grammar = analyze(source)?;
let tree = parse_input(&grammar, test_input)?;
Ok(AnalyzeResult {
grammar_name: grammar.name,
tree,
})
}
pub fn parse_input(grammar: &Grammar, input: &str) -> Result<ParseTree, RantlrError> {
let engine = Engine::from_grammar(grammar).map_err(|msg| {
RantlrError::parse(input, msg, Span::from_offsets(0, 0))
})?;
engine.parse(input)
}
fn diagnostic_from_error(_source: &str, err: &RantlrError) -> DiagnosticJson {
let end_col = err.column.saturating_add(1);
DiagnosticJson {
severity: "Error".into(),
kind: format!("{:?}", err.kind),
line: err.line,
column: err.column,
end_line: err.line,
end_column: end_col,
message: err.message.clone(),
help: err.help.clone(),
code: err.code.clone(),
fix_hint: err.fix_hint.clone(),
}
}
fn diagnostic_from_diag(source: &str, diag: &Diagnostic) -> DiagnosticJson {
let (line, column) = diag.span.line_col(source);
let end = diag.span.end.as_usize().min(source.len());
let (end_line, end_column) = Span::from_offsets(end, end).line_col(source);
DiagnosticJson {
severity: format!("{:?}", diag.severity),
kind: format!("{:?}", diag.kind),
line,
column,
end_line,
end_column: end_column.max(column + 1),
message: diag.message.clone(),
help: diag.help.clone(),
code: diag.code.clone(),
fix_hint: diag.fix.clone(),
}
}
struct TokDef {
name: String,
re: Regex,
skip: bool,
}
struct Engine<'g> {
rules: BTreeMap<String, &'g RuleDecl>,
start: String,
tokens: Vec<TokDef>,
}
#[derive(Clone)]
struct LexTok {
name: String,
text: String,
}
struct Parser<'a> {
tokens: &'a [LexTok],
pos: usize,
rules: &'a BTreeMap<String, &'a RuleDecl>,
}
impl<'g> Engine<'g> {
fn from_grammar(grammar: &'g Grammar) -> Result<Self, String> {
let mut rules = BTreeMap::new();
let mut start = None;
let mut declared = Vec::new();
for item in &grammar.items {
match item {
GrammarItem::Rule(r) => {
if start.is_none() {
start = Some(r.name.clone());
}
rules.insert(r.name.clone(), r);
}
GrammarItem::Token(t) => {
declared.push((t.name.clone(), pattern_for_body(&t.body)?, t.skip));
}
_ => {}
}
}
let start = start.ok_or_else(|| "grammar has no rules".to_string())?;
let mut inferred: BTreeMap<String, String> = BTreeMap::new();
for rule in rules.values() {
collect_literals(&rule.body, &mut inferred);
}
let mut tokens = Vec::new();
for (name, pat, skip) in declared {
tokens.push(TokDef {
name,
re: Regex::new(&format!("^{pat}")).map_err(|e| e.to_string())?,
skip,
});
}
for (lit, name) in inferred {
if tokens.iter().any(|t| t.name == name) {
continue;
}
let pat = regex_escape(&lit);
tokens.push(TokDef {
name,
re: Regex::new(&format!("^{pat}")).map_err(|e| e.to_string())?,
skip: false,
});
}
tokens.sort_by(|a, b| {
token_specificity(b).cmp(&token_specificity(a)).then_with(|| {
b.re.as_str().len().cmp(&a.re.as_str().len())
})
});
Ok(Self {
rules,
start,
tokens,
})
}
fn parse(&self, input: &str) -> Result<ParseTree, RantlrError> {
let lexed = self.tokenize(input)?;
let mut parser = Parser {
tokens: &lexed,
pos: 0,
rules: &self.rules,
};
let tree = parser.parse_rule(&self.start)?;
if !parser.at_end() {
let t = parser.peek();
return Err(RantlrError::parse(
input,
format!("unexpected trailing input (`{}`)", t.name),
Span::from_offsets(0, 0),
));
}
Ok(tree)
}
fn tokenize(&self, input: &str) -> Result<Vec<LexTok>, RantlrError> {
let mut out = Vec::new();
let mut i = 0;
let bytes = input.as_bytes();
while i < input.len() {
let rest = &input[i..];
let mut best: Option<(usize, i32, &TokDef)> = None;
for tok in &self.tokens {
if let Some(m) = tok.re.find(rest) {
if m.start() == 0 {
let len = m.end();
let spec = token_specificity(tok);
let better = match best {
None => true,
Some((bl, bs, _)) => len > bl || (len == bl && spec > bs),
};
if better {
best = Some((len, spec, tok));
}
}
}
}
let Some((len, _, tok)) = best else {
let (line, col) = Span::from_offsets(i, i).line_col(input);
return Err(RantlrError {
kind: crate::error::ErrorKind::Parse,
message: format!(
"unexpected character `{}`",
input[i..].chars().next().unwrap_or('?')
),
line,
column: col,
snippet: crate::error::context_snippet(input, Span::from_offsets(i, i + 1)),
help: None,
code: None,
fix_hint: None,
});
};
if !tok.skip {
out.push(LexTok {
name: tok.name.clone(),
text: input[i..i + len].to_string(),
});
}
i += len;
let _ = bytes; }
out.push(LexTok {
name: "EOF".into(),
text: String::new(),
});
Ok(out)
}
}
impl<'a> Parser<'a> {
fn at_end(&self) -> bool {
self.peek().name == "EOF"
}
fn peek(&self) -> &LexTok {
&self.tokens[self.pos.min(self.tokens.len() - 1)]
}
fn mark(&self) -> usize {
self.pos
}
fn reset(&mut self, m: usize) {
self.pos = m;
}
fn expect_token(&mut self, name: &str) -> Result<ParseTree, RantlrError> {
let tok = self.peek().clone();
if tok.name == name {
if tok.name != "EOF" {
self.pos += 1;
}
return Ok(ParseTree {
kind: name.into(),
text: tok.text,
children: vec![],
});
}
Err(self.err(format!("expected `{name}`, found `{}`", tok.name)))
}
fn parse_rule(&mut self, name: &str) -> Result<ParseTree, RantlrError> {
let rule = self
.rules
.get(name)
.copied()
.ok_or_else(|| self.err(format!("unknown rule `{name}`")))?;
let children = self.parse_expr(&rule.body)?;
let text = children.iter().map(|c| c.text.as_str()).collect::<String>();
Ok(ParseTree {
kind: name.into(),
text,
children,
})
}
fn parse_expr(&mut self, expr: &Expr) -> Result<Vec<ParseTree>, RantlrError> {
match expr {
Expr::Seq { items } => {
let mut out = Vec::new();
for item in items {
out.extend(self.parse_expr(item)?);
}
Ok(out)
}
Expr::Group { body } => self.parse_expr(body),
Expr::Ref { name, .. } => {
if self.rules.contains_key(name) {
Ok(vec![self.parse_rule(name)?])
} else {
Ok(vec![self.expect_token(name)?])
}
}
Expr::Literal { value, .. } => {
let tok_name = literal_token_name(value);
Ok(vec![self.expect_token(&tok_name)?])
}
Expr::Optional { body } => {
let m = self.mark();
match self.parse_expr(body) {
Ok(nodes) => Ok(nodes),
Err(_) => {
self.reset(m);
Ok(vec![])
}
}
}
Expr::Repeat { min, max, body } => {
let mut out = Vec::new();
let mut n = 0u64;
loop {
if let Some(max) = *max {
if n >= max {
break;
}
}
let m = self.mark();
match self.parse_expr(body) {
Ok(nodes) => {
out.extend(nodes);
n += 1;
}
Err(_) => {
self.reset(m);
break;
}
}
}
if let Some(min) = *min {
if n < min {
return Err(self.err(format!("expected at least {min} repetitions")));
}
}
Ok(out)
}
Expr::Match { arms } | Expr::Alt { alts: arms } => {
let mut last_err = None;
for arm in arms {
let m = self.mark();
match self.parse_expr(arm) {
Ok(nodes) => return Ok(nodes),
Err(e) => {
self.reset(m);
last_err = Some(e);
}
}
}
Err(last_err.unwrap_or_else(|| self.err("no alternative matched")))
}
}
}
fn err(&self, message: impl Into<String>) -> RantlrError {
RantlrError {
kind: crate::error::ErrorKind::Parse,
message: message.into(),
line: 1,
column: 1,
snippet: String::new(),
help: Some(format!("at token `{}`", self.peek().name)),
code: None,
fix_hint: None,
}
}
}
fn pattern_for_body(body: &TokenBody) -> Result<String, String> {
Ok(match body {
TokenBody::Literal(s) => regex_escape(s),
TokenBody::Builtin(b) => builtin_regex(*b).to_string(),
})
}
fn token_specificity(tok: &TokDef) -> i32 {
if tok.name.starts_with("Lit_") {
return 100;
}
let pat = tok.re.as_str();
if pat.contains("[A-Za-z_") || pat.contains("[0-9]") {
return 10;
}
50
}
fn builtin_regex(b: BuiltinType) -> &'static str {
match b {
BuiltinType::Number => r"[0-9]+(?:\.[0-9]+)?",
BuiltinType::QuotedString => r#"(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')"#,
BuiltinType::Email => r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}",
BuiltinType::Url => r"https?://[^\s]+",
BuiltinType::DateTime => r"\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}:\d{2})?",
BuiltinType::Identifier => r"[A-Za-z_][A-Za-z0-9_]*",
}
}
fn regex_escape(s: &str) -> String {
let mut out = String::new();
for c in s.chars() {
if matches!(
c,
'\\' | '.' | '+' | '*' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '^' | '$' | '|'
) {
out.push('\\');
}
out.push(c);
}
out
}
fn collect_literals(expr: &Expr, out: &mut BTreeMap<String, String>) {
match expr {
Expr::Alt { alts } | Expr::Seq { items: alts } | Expr::Match { arms: alts } => {
for e in alts {
collect_literals(e, out);
}
}
Expr::Repeat { body, .. } | Expr::Optional { body } | Expr::Group { body } => {
collect_literals(body, out);
}
Expr::Literal { value, .. } => {
out.entry(value.clone())
.or_insert_with(|| literal_token_name(value));
}
Expr::Ref { .. } => {}
}
}
fn literal_token_name(lit: &str) -> String {
let mapped: String = lit
.chars()
.map(|c| match c {
'+' => "Plus".into(),
'-' => "Minus".into(),
'*' => "Star".into(),
'/' => "Slash".into(),
'(' => "LParen".into(),
')' => "RParen".into(),
c if c.is_ascii_alphanumeric() => c.to_string(),
_ => format!("U{:04X}", c as u32),
})
.collect();
format!("Lit_{mapped}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn interprets_calculator() {
let src = include_str!("../testdata/calculator.gr");
let result = analyze_and_parse(src, "1+2*3").unwrap();
assert_eq!(result.grammar_name, "Calculator");
assert_eq!(result.tree.kind, "prog");
assert_eq!(result.tree.text, "1+2*3");
}
#[test]
fn diagnose_left_recursion() {
let src = include_str!("../testdata/left_recursive.gr");
let diags = diagnose(src);
assert_eq!(diags.len(), 1);
assert_eq!(diags[0].code.as_deref(), Some("left-recursion"));
}
#[test]
fn interprets_api_gateway() {
let src = include_str!("../testdata/api_gateway.gr");
let input = r#"gateway api_v1 { route /users/:id { methods GET POST limit 10000 per ip balance RoundRobin auth jwt oauth nested posts { constraint "^[a-z0-9-]{3,40}$" methods GET limit 5000 per ip balance LeastConn } } }"#;
let result = analyze_and_parse(src, input).expect("api gateway");
assert_eq!(result.grammar_name, "ApiGateway");
assert_eq!(result.tree.kind, "gateway");
}
}