use serde_json::Value;
use crate::errors::HaysonQueryParseError;
use super::planner::QuerySpec;
use super::predicates::{Combine, ComparisonOp, PredicateNode};
#[derive(Debug, Clone, PartialEq)]
enum Token {
Ident(String),
String(String),
Number(String),
Op(&'static str),
LParen,
RParen,
LBracket,
RBracket,
Comma,
Eof,
}
#[derive(Debug, Clone, PartialEq)]
struct Spanned {
token: Token,
position: usize,
}
fn err(position: usize, reason: impl Into<String>) -> HaysonQueryParseError {
HaysonQueryParseError {
position,
reason: reason.into(),
}
}
fn tokenize(text: &str) -> Result<Vec<Spanned>, HaysonQueryParseError> {
let bytes = text.as_bytes();
let mut tokens = Vec::new();
let mut i = 0usize;
while i < bytes.len() {
let c = bytes[i] as char;
if c.is_whitespace() {
i += 1;
continue;
}
let start = i;
match c {
'(' => {
tokens.push(Spanned {
token: Token::LParen,
position: start,
});
i += 1;
}
')' => {
tokens.push(Spanned {
token: Token::RParen,
position: start,
});
i += 1;
}
'[' => {
tokens.push(Spanned {
token: Token::LBracket,
position: start,
});
i += 1;
}
']' => {
tokens.push(Spanned {
token: Token::RBracket,
position: start,
});
i += 1;
}
',' => {
tokens.push(Spanned {
token: Token::Comma,
position: start,
});
i += 1;
}
'=' => {
tokens.push(Spanned {
token: Token::Op("="),
position: start,
});
i += 1;
}
'!' => {
if bytes.get(i + 1) == Some(&b'=') {
tokens.push(Spanned {
token: Token::Op("!="),
position: start,
});
i += 2;
} else {
return Err(err(start, "expected \"!=\", found a bare \"!\""));
}
}
'>' => {
if bytes.get(i + 1) == Some(&b'=') {
tokens.push(Spanned {
token: Token::Op(">="),
position: start,
});
i += 2;
} else {
tokens.push(Spanned {
token: Token::Op(">"),
position: start,
});
i += 1;
}
}
'<' => {
if bytes.get(i + 1) == Some(&b'=') {
tokens.push(Spanned {
token: Token::Op("<="),
position: start,
});
i += 2;
} else {
tokens.push(Spanned {
token: Token::Op("<"),
position: start,
});
i += 1;
}
}
'\'' | '"' => {
let quote = c;
let mut value = String::new();
i += 1;
let mut closed = false;
while i < bytes.len() {
let ch = bytes[i] as char;
if ch == quote {
i += 1;
closed = true;
break;
}
if ch == '\\' && i + 1 < bytes.len() {
value.push(bytes[i + 1] as char);
i += 2;
continue;
}
value.push(ch);
i += 1;
}
if !closed {
return Err(err(start, "unterminated string literal"));
}
tokens.push(Spanned {
token: Token::String(value),
position: start,
});
}
_ if c.is_ascii_digit() || (c == '-' && bytes.get(i + 1).is_some_and(u8::is_ascii_digit)) => {
let mut end = i + 1;
while end < bytes.len() && (bytes[end].is_ascii_digit() || bytes[end] == b'.') {
end += 1;
}
let slice = &text[start..end];
tokens.push(Spanned {
token: Token::Number(slice.to_string()),
position: start,
});
i = end;
}
_ if c.is_alphabetic() || c == '_' => {
let mut end = i + 1;
while end < bytes.len() && ((bytes[end] as char).is_alphanumeric() || bytes[end] == b'_') {
end += 1;
}
tokens.push(Spanned {
token: Token::Ident(text[start..end].to_string()),
position: start,
});
i = end;
}
other => return Err(err(start, format!("unexpected character \"{other}\""))),
}
}
let eof_position = bytes.len();
tokens.push(Spanned {
token: Token::Eof,
position: eof_position,
});
Ok(tokens)
}
struct Parser<'a> {
tokens: &'a [Spanned],
pos: usize,
}
impl<'a> Parser<'a> {
fn peek(&self) -> &Spanned {
&self.tokens[self.pos]
}
fn advance(&mut self) -> Spanned {
let token = self.tokens[self.pos].clone();
if self.pos + 1 < self.tokens.len() {
self.pos += 1;
}
token
}
fn expect_ident_keyword(&mut self, keyword: &str) -> bool {
if let Token::Ident(name) = &self.peek().token {
if name.eq_ignore_ascii_case(keyword) {
self.advance();
return true;
}
}
false
}
fn parse_expr(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
let mut terms = vec![self.parse_and()?];
while self.expect_ident_keyword("or") {
terms.push(self.parse_and()?);
}
Ok(if terms.len() == 1 {
terms.pop().unwrap()
} else {
PredicateNode::Group {
combine: Combine::Or,
nodes: terms,
}
})
}
fn parse_and(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
let mut terms = vec![self.parse_unary()?];
while self.expect_ident_keyword("and") {
terms.push(self.parse_unary()?);
}
Ok(if terms.len() == 1 {
terms.pop().unwrap()
} else {
PredicateNode::Group {
combine: Combine::And,
nodes: terms,
}
})
}
fn parse_unary(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
if matches!(self.peek().token, Token::LParen) {
self.advance();
let node = self.parse_expr()?;
let closing = self.peek().clone();
if !matches!(closing.token, Token::RParen) {
return Err(err(
closing.position,
"expected \")\" to close the bracket opened earlier",
));
}
self.advance();
return Ok(node);
}
self.parse_comparison()
}
fn parse_comparison(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
let field_token = self.peek().clone();
let field = match field_token.token {
Token::Ident(name) => {
self.advance();
name
}
other => {
return Err(err(
field_token.position,
format!("expected a field name, found {}", describe(&other)),
))
}
};
if self.expect_ident_keyword("in") {
let value = self.parse_value()?;
if !matches!(value, Value::Array(_)) {
return Err(err(
field_token.position,
"\"in\" needs an array of values, e.g. field in [1, 2]",
));
}
return Ok(PredicateNode::leaf(field, ComparisonOp::In, value));
}
if self.expect_ident_keyword("contains") {
let value = self.parse_value()?;
return Ok(PredicateNode::leaf(field, ComparisonOp::Contains, value));
}
if self.expect_ident_keyword("startswith") {
let value = self.parse_value()?;
return Ok(PredicateNode::leaf(field, ComparisonOp::StartsWith, value));
}
let op_token = self.peek().clone();
let op = match op_token.token {
Token::Op("=") => ComparisonOp::Eq,
Token::Op("!=") => ComparisonOp::Ne,
Token::Op(">") => ComparisonOp::Gt,
Token::Op(">=") => ComparisonOp::Gte,
Token::Op("<") => ComparisonOp::Lt,
Token::Op("<=") => ComparisonOp::Lte,
ref other => {
return Err(err(
op_token.position,
format!(
"expected a comparison operator (= != > >= < <= in contains startsWith), found {}",
describe(other)
),
))
}
};
self.advance();
let value = self.parse_value()?;
Ok(PredicateNode::leaf(field, op, value))
}
fn parse_value(&mut self) -> Result<Value, HaysonQueryParseError> {
let token = self.peek().clone();
match token.token {
Token::String(text) => {
self.advance();
Ok(Value::String(text))
}
Token::Number(text) => {
self.advance();
serde_json::from_str(&text)
.map_err(|_| err(token.position, format!("\"{text}\" is not a valid number")))
}
Token::Ident(name) if name.eq_ignore_ascii_case("true") => {
self.advance();
Ok(Value::Bool(true))
}
Token::Ident(name) if name.eq_ignore_ascii_case("false") => {
self.advance();
Ok(Value::Bool(false))
}
Token::Ident(name) if name.eq_ignore_ascii_case("null") => {
self.advance();
Ok(Value::Null)
}
Token::LBracket => {
self.advance();
let mut items = Vec::new();
if !matches!(self.peek().token, Token::RBracket) {
items.push(self.parse_value()?);
while matches!(self.peek().token, Token::Comma) {
self.advance();
items.push(self.parse_value()?);
}
}
let closing = self.peek().clone();
if !matches!(closing.token, Token::RBracket) {
return Err(err(closing.position, "expected \"]\" to close the array"));
}
self.advance();
Ok(Value::Array(items))
}
other => Err(err(
token.position,
format!(
"expected a value (string, number, true, false, null or array), found {}",
describe(&other)
),
)),
}
}
}
fn describe(token: &Token) -> String {
match token {
Token::Ident(name) => format!("identifier \"{name}\""),
Token::String(text) => format!("string \"{text}\""),
Token::Number(text) => format!("number {text}"),
Token::Op(op) => format!("operator \"{op}\""),
Token::LParen => "\"(\"".to_string(),
Token::RParen => "\")\"".to_string(),
Token::LBracket => "\"[\"".to_string(),
Token::RBracket => "\"]\"".to_string(),
Token::Comma => "\",\"".to_string(),
Token::Eof => "end of query".to_string(),
}
}
pub fn parse_query(text: &str) -> Result<QuerySpec, HaysonQueryParseError> {
let tokens = tokenize(text)?;
let mut parser = Parser {
tokens: &tokens,
pos: 0,
};
let root = parser.parse_expr()?;
let trailing = parser.peek().clone();
if !matches!(trailing.token, Token::Eof) {
return Err(err(
trailing.position,
format!("unexpected {} after a complete expression", describe(&trailing.token)),
));
}
let root = match root {
PredicateNode::Group {
combine: Combine::And,
nodes,
} => PredicateNode::Group {
combine: Combine::And,
nodes,
},
other => PredicateNode::Group {
combine: Combine::And,
nodes: vec![other],
},
};
Ok(QuerySpec::new(root))
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
use crate::query::builder::QueryBuilder;
#[test]
fn parses_a_single_comparison() {
let spec = parse_query("age >= 30").unwrap();
let expected = QueryBuilder::new().r#where("age", ComparisonOp::Gte, json!(30)).spec();
assert_eq!(spec, expected);
}
#[test]
fn parses_every_comparison_operator() {
let cases: &[(&str, ComparisonOp, Value)] = &[
("role = 'admin'", ComparisonOp::Eq, json!("admin")),
("role != 'admin'", ComparisonOp::Ne, json!("admin")),
("age > 30", ComparisonOp::Gt, json!(30)),
("age >= 30", ComparisonOp::Gte, json!(30)),
("age < 30", ComparisonOp::Lt, json!(30)),
("age <= 30", ComparisonOp::Lte, json!(30)),
("age in [1, 2, 3]", ComparisonOp::In, json!([1, 2, 3])),
("tags contains 'go'", ComparisonOp::Contains, json!("go")),
("name startsWith 'An'", ComparisonOp::StartsWith, json!("An")),
];
for (text, op, value) in cases {
let spec = parse_query(text).unwrap();
let field = text.split_whitespace().next().unwrap();
let expected = QueryBuilder::new().r#where(field, *op, value.clone()).spec();
assert_eq!(spec, expected, "mismatch for {text:?}");
}
}
#[test]
fn flat_and_matches_chained_where() {
let spec = parse_query("role = 'admin' AND age > 30").unwrap();
let expected = QueryBuilder::new()
.r#where("role", ComparisonOp::Eq, json!("admin"))
.r#where("age", ComparisonOp::Gt, json!(30))
.spec();
assert_eq!(spec, expected);
}
#[test]
fn and_inside_or_bracket_matches_where_any_where_all() {
let text = "role = 'admin' OR (role = 'user' AND age > 35)";
let spec = parse_query(text).unwrap();
let expected = QueryBuilder::new()
.where_any(|group| {
group
.r#where("role", ComparisonOp::Eq, json!("admin"))
.where_all(|inner| {
inner.r#where("role", ComparisonOp::Eq, json!("user")).r#where(
"age",
ComparisonOp::Gt,
json!(35),
);
});
})
.spec();
assert_eq!(spec, expected);
}
#[test]
fn or_bracket_inside_flat_and_matches_where_then_where_any() {
let text = "role = 'user' AND (tags contains 'go' OR tags contains 'node')";
let spec = parse_query(text).unwrap();
let expected = QueryBuilder::new()
.r#where("role", ComparisonOp::Eq, json!("user"))
.where_any(|group| {
group.r#where("tags", ComparisonOp::Contains, json!("go")).r#where(
"tags",
ComparisonOp::Contains,
json!("node"),
);
})
.spec();
assert_eq!(spec, expected);
}
#[test]
fn rejects_garbage_with_a_positioned_error() {
let error = parse_query("age >=").unwrap_err();
assert_eq!(error.position, 6);
let error = parse_query("age 30").unwrap_err();
assert!(error.reason.contains("comparison operator"));
let error = parse_query("age >= 30 age").unwrap_err();
assert!(error.reason.contains("unexpected"));
let error = parse_query("(age >= 30").unwrap_err();
assert!(error.reason.contains("\")\""));
let error = parse_query("age in 30").unwrap_err();
assert!(error.reason.contains("array"));
}
#[test]
fn error_display_carries_position_and_reason() {
let error = parse_query("").unwrap_err();
let message = error.to_string();
assert!(message.contains("position"));
assert!(message.contains(&error.reason));
}
}