use serde_json::Value;
use crate::errors::HaysonQueryParseError;
use super::planner::{QuerySpec, SortDirection};
use super::predicates::{Combine, ComparisonOp, PredicateNode};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum Clause {
OrderBy,
Limit,
Offset,
}
impl Clause {
fn name(self) -> &'static str {
match self {
Clause::OrderBy => "ORDER BY",
Clause::Limit => "LIMIT",
Clause::Offset => "OFFSET",
}
}
}
#[derive(Debug, Clone, PartialEq)]
enum Token {
Ident(String),
String(String),
Number(String),
Op(&'static str),
LParen,
RParen,
LBracket,
RBracket,
Comma,
Eof,
}
#[derive(Debug, Clone, PartialEq)]
struct Spanned {
token: Token,
position: usize,
}
fn err(position: usize, reason: impl Into<String>) -> HaysonQueryParseError {
HaysonQueryParseError {
position,
reason: reason.into(),
}
}
fn tokenize(text: &str) -> Result<Vec<Spanned>, HaysonQueryParseError> {
let bytes = text.as_bytes();
let mut tokens = Vec::new();
let mut i = 0usize;
while i < bytes.len() {
let c = bytes[i] as char;
if c.is_whitespace() {
i += 1;
continue;
}
let start = i;
match c {
'(' => {
tokens.push(Spanned {
token: Token::LParen,
position: start,
});
i += 1;
}
')' => {
tokens.push(Spanned {
token: Token::RParen,
position: start,
});
i += 1;
}
'[' => {
tokens.push(Spanned {
token: Token::LBracket,
position: start,
});
i += 1;
}
']' => {
tokens.push(Spanned {
token: Token::RBracket,
position: start,
});
i += 1;
}
',' => {
tokens.push(Spanned {
token: Token::Comma,
position: start,
});
i += 1;
}
'=' => {
tokens.push(Spanned {
token: Token::Op("="),
position: start,
});
i += 1;
}
'!' => {
if bytes.get(i + 1) == Some(&b'=') {
tokens.push(Spanned {
token: Token::Op("!="),
position: start,
});
i += 2;
} else {
return Err(err(start, "expected \"!=\", found a bare \"!\""));
}
}
'>' => {
if bytes.get(i + 1) == Some(&b'=') {
tokens.push(Spanned {
token: Token::Op(">="),
position: start,
});
i += 2;
} else {
tokens.push(Spanned {
token: Token::Op(">"),
position: start,
});
i += 1;
}
}
'<' => {
if bytes.get(i + 1) == Some(&b'=') {
tokens.push(Spanned {
token: Token::Op("<="),
position: start,
});
i += 2;
} else {
tokens.push(Spanned {
token: Token::Op("<"),
position: start,
});
i += 1;
}
}
'\'' | '"' => {
let quote = c;
let mut value = String::new();
i += 1;
let mut closed = false;
while i < bytes.len() {
let ch = bytes[i] as char;
if ch == quote {
i += 1;
closed = true;
break;
}
if ch == '\\' && i + 1 < bytes.len() {
value.push(bytes[i + 1] as char);
i += 2;
continue;
}
value.push(ch);
i += 1;
}
if !closed {
return Err(err(start, "unterminated string literal"));
}
tokens.push(Spanned {
token: Token::String(value),
position: start,
});
}
_ if c.is_ascii_digit() || (c == '-' && bytes.get(i + 1).is_some_and(u8::is_ascii_digit)) => {
let mut end = i + 1;
while end < bytes.len() && (bytes[end].is_ascii_digit() || bytes[end] == b'.') {
end += 1;
}
let slice = &text[start..end];
tokens.push(Spanned {
token: Token::Number(slice.to_string()),
position: start,
});
i = end;
}
_ if c.is_alphabetic() || c == '_' => {
let mut end = i + 1;
while end < bytes.len() && ((bytes[end] as char).is_alphanumeric() || bytes[end] == b'_') {
end += 1;
}
tokens.push(Spanned {
token: Token::Ident(text[start..end].to_string()),
position: start,
});
i = end;
}
other => return Err(err(start, format!("unexpected character \"{other}\""))),
}
}
let eof_position = bytes.len();
tokens.push(Spanned {
token: Token::Eof,
position: eof_position,
});
Ok(tokens)
}
struct Parser<'a> {
tokens: &'a [Spanned],
pos: usize,
}
impl<'a> Parser<'a> {
fn peek(&self) -> &Spanned {
&self.tokens[self.pos]
}
fn advance(&mut self) -> Spanned {
let token = self.tokens[self.pos].clone();
if self.pos + 1 < self.tokens.len() {
self.pos += 1;
}
token
}
fn expect_ident_keyword(&mut self, keyword: &str) -> bool {
if let Token::Ident(name) = &self.peek().token {
if name.eq_ignore_ascii_case(keyword) {
self.advance();
return true;
}
}
false
}
fn peek_at(&self, ahead: usize) -> &Token {
let last = self.tokens.len() - 1;
&self.tokens[(self.pos + ahead).min(last)].token
}
fn peek_clause(&self) -> Option<Clause> {
let Token::Ident(name) = &self.peek().token else {
return None;
};
if name.eq_ignore_ascii_case("order") {
Some(Clause::OrderBy)
} else if name.eq_ignore_ascii_case("limit") {
Some(Clause::Limit)
} else if name.eq_ignore_ascii_case("offset") {
Some(Clause::Offset)
} else {
None
}
}
fn starts_with_clause(&self) -> bool {
match self.peek_clause() {
Some(Clause::OrderBy) => {
matches!(self.peek_at(1), Token::Ident(next) if next.eq_ignore_ascii_case("by"))
}
Some(_) => match self.peek_at(1) {
Token::Op(_) => false,
Token::Ident(next) => !["in", "contains", "startswith"]
.iter()
.any(|word| next.eq_ignore_ascii_case(word)),
_ => true,
},
None => false,
}
}
fn parse_clauses(&mut self, mut spec: QuerySpec) -> Result<QuerySpec, HaysonQueryParseError> {
let mut previous: Option<Clause> = None;
while let Some(clause) = self.peek_clause() {
let position = self.peek().position;
match previous {
Some(seen) if seen == clause => {
return Err(err(
position,
format!("\"{}\" given twice — each clause may appear once", clause.name()),
))
}
Some(seen) if clause < seen => {
return Err(err(
position,
format!("\"{}\" must come before \"{}\"", clause.name(), seen.name()),
))
}
_ => {}
}
self.advance();
spec = match clause {
Clause::OrderBy => {
let by = self.peek().clone();
if !self.expect_ident_keyword("by") {
return Err(err(
by.position,
format!("expected \"BY\" after \"ORDER\", found {}", describe(&by.token)),
));
}
let field_token = self.peek().clone();
let Token::Ident(field) = field_token.token else {
return Err(err(
field_token.position,
format!(
"expected a field name after \"ORDER BY\", found {}",
describe(&field_token.token)
),
));
};
self.advance();
let direction = if self.expect_ident_keyword("desc") {
SortDirection::Desc
} else {
self.expect_ident_keyword("asc");
SortDirection::Asc
};
spec.with_order_by(field, direction)
}
Clause::Limit => spec.with_limit(self.parse_count(clause)?),
Clause::Offset => spec.with_offset(self.parse_count(clause)?),
};
previous = Some(clause);
}
Ok(spec)
}
fn parse_count(&mut self, clause: Clause) -> Result<usize, HaysonQueryParseError> {
let token = self.peek().clone();
let parsed = match &token.token {
Token::Number(text) => text.parse::<usize>().ok(),
_ => None,
};
let Some(count) = parsed else {
return Err(err(
token.position,
format!(
"\"{}\" needs a non-negative integer, found {}",
clause.name(),
describe(&token.token)
),
));
};
self.advance();
Ok(count)
}
fn parse_expr(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
let mut terms = vec![self.parse_and()?];
while self.expect_ident_keyword("or") {
terms.push(self.parse_and()?);
}
Ok(if terms.len() == 1 {
terms.pop().unwrap()
} else {
PredicateNode::Group {
combine: Combine::Or,
nodes: terms,
}
})
}
fn parse_and(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
let mut terms = vec![self.parse_unary()?];
while self.expect_ident_keyword("and") {
terms.push(self.parse_unary()?);
}
Ok(if terms.len() == 1 {
terms.pop().unwrap()
} else {
PredicateNode::Group {
combine: Combine::And,
nodes: terms,
}
})
}
fn parse_unary(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
if matches!(self.peek().token, Token::LParen) {
self.advance();
let node = self.parse_expr()?;
let closing = self.peek().clone();
if !matches!(closing.token, Token::RParen) {
return Err(err(
closing.position,
"expected \")\" to close the bracket opened earlier",
));
}
self.advance();
return Ok(node);
}
self.parse_comparison()
}
fn parse_comparison(&mut self) -> Result<PredicateNode, HaysonQueryParseError> {
let field_token = self.peek().clone();
let field = match field_token.token {
Token::Ident(name) => {
self.advance();
name
}
other => {
return Err(err(
field_token.position,
format!("expected a field name, found {}", describe(&other)),
))
}
};
if self.expect_ident_keyword("in") {
let value = self.parse_value()?;
if !matches!(value, Value::Array(_)) {
return Err(err(
field_token.position,
"\"in\" needs an array of values, e.g. field in [1, 2]",
));
}
return Ok(PredicateNode::leaf(field, ComparisonOp::In, value));
}
if self.expect_ident_keyword("contains") {
let value = self.parse_value()?;
return Ok(PredicateNode::leaf(field, ComparisonOp::Contains, value));
}
if self.expect_ident_keyword("startswith") {
let value = self.parse_value()?;
return Ok(PredicateNode::leaf(field, ComparisonOp::StartsWith, value));
}
let op_token = self.peek().clone();
let op = match op_token.token {
Token::Op("=") => ComparisonOp::Eq,
Token::Op("!=") => ComparisonOp::Ne,
Token::Op(">") => ComparisonOp::Gt,
Token::Op(">=") => ComparisonOp::Gte,
Token::Op("<") => ComparisonOp::Lt,
Token::Op("<=") => ComparisonOp::Lte,
ref other => {
return Err(err(
op_token.position,
format!(
"expected a comparison operator (= != > >= < <= in contains startsWith), found {}",
describe(other)
),
))
}
};
self.advance();
let value = self.parse_value()?;
Ok(PredicateNode::leaf(field, op, value))
}
fn parse_value(&mut self) -> Result<Value, HaysonQueryParseError> {
let token = self.peek().clone();
match token.token {
Token::String(text) => {
self.advance();
Ok(Value::String(text))
}
Token::Number(text) => {
self.advance();
serde_json::from_str(&text)
.map_err(|_| err(token.position, format!("\"{text}\" is not a valid number")))
}
Token::Ident(name) if name.eq_ignore_ascii_case("true") => {
self.advance();
Ok(Value::Bool(true))
}
Token::Ident(name) if name.eq_ignore_ascii_case("false") => {
self.advance();
Ok(Value::Bool(false))
}
Token::Ident(name) if name.eq_ignore_ascii_case("null") => {
self.advance();
Ok(Value::Null)
}
Token::LBracket => {
self.advance();
let mut items = Vec::new();
if !matches!(self.peek().token, Token::RBracket) {
items.push(self.parse_value()?);
while matches!(self.peek().token, Token::Comma) {
self.advance();
items.push(self.parse_value()?);
}
}
let closing = self.peek().clone();
if !matches!(closing.token, Token::RBracket) {
return Err(err(closing.position, "expected \"]\" to close the array"));
}
self.advance();
Ok(Value::Array(items))
}
other => Err(err(
token.position,
format!(
"expected a value (string, number, true, false, null or array), found {}",
describe(&other)
),
)),
}
}
}
fn describe(token: &Token) -> String {
match token {
Token::Ident(name) => format!("identifier \"{name}\""),
Token::String(text) => format!("string \"{text}\""),
Token::Number(text) => format!("number {text}"),
Token::Op(op) => format!("operator \"{op}\""),
Token::LParen => "\"(\"".to_string(),
Token::RParen => "\")\"".to_string(),
Token::LBracket => "\"[\"".to_string(),
Token::RBracket => "\"]\"".to_string(),
Token::Comma => "\",\"".to_string(),
Token::Eof => "end of query".to_string(),
}
}
pub fn parse_query(text: &str) -> Result<QuerySpec, HaysonQueryParseError> {
let tokens = tokenize(text)?;
let mut parser = Parser {
tokens: &tokens,
pos: 0,
};
let root = if parser.starts_with_clause() {
PredicateNode::Group {
combine: Combine::And,
nodes: Vec::new(),
}
} else {
match parser.parse_expr()? {
PredicateNode::Group {
combine: Combine::And,
nodes,
} => PredicateNode::Group {
combine: Combine::And,
nodes,
},
other => PredicateNode::Group {
combine: Combine::And,
nodes: vec![other],
},
}
};
let spec = parser.parse_clauses(QuerySpec::new(root))?;
let trailing = parser.peek().clone();
if !matches!(trailing.token, Token::Eof) {
return Err(err(
trailing.position,
format!("unexpected {} after a complete expression", describe(&trailing.token)),
));
}
Ok(spec)
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
use crate::query::builder::QueryBuilder;
#[test]
fn parses_a_single_comparison() {
let spec = parse_query("age >= 30").unwrap();
let expected = QueryBuilder::new().r#where("age", ComparisonOp::Gte, json!(30)).spec();
assert_eq!(spec, expected);
}
#[test]
fn parses_every_comparison_operator() {
let cases: &[(&str, ComparisonOp, Value)] = &[
("role = 'admin'", ComparisonOp::Eq, json!("admin")),
("role != 'admin'", ComparisonOp::Ne, json!("admin")),
("age > 30", ComparisonOp::Gt, json!(30)),
("age >= 30", ComparisonOp::Gte, json!(30)),
("age < 30", ComparisonOp::Lt, json!(30)),
("age <= 30", ComparisonOp::Lte, json!(30)),
("age in [1, 2, 3]", ComparisonOp::In, json!([1, 2, 3])),
("tags contains 'go'", ComparisonOp::Contains, json!("go")),
("name startsWith 'An'", ComparisonOp::StartsWith, json!("An")),
];
for (text, op, value) in cases {
let spec = parse_query(text).unwrap();
let field = text.split_whitespace().next().unwrap();
let expected = QueryBuilder::new().r#where(field, *op, value.clone()).spec();
assert_eq!(spec, expected, "mismatch for {text:?}");
}
}
#[test]
fn flat_and_matches_chained_where() {
let spec = parse_query("role = 'admin' AND age > 30").unwrap();
let expected = QueryBuilder::new()
.r#where("role", ComparisonOp::Eq, json!("admin"))
.r#where("age", ComparisonOp::Gt, json!(30))
.spec();
assert_eq!(spec, expected);
}
#[test]
fn and_inside_or_bracket_matches_where_any_where_all() {
let text = "role = 'admin' OR (role = 'user' AND age > 35)";
let spec = parse_query(text).unwrap();
let expected = QueryBuilder::new()
.where_any(|group| {
group
.r#where("role", ComparisonOp::Eq, json!("admin"))
.where_all(|inner| {
inner.r#where("role", ComparisonOp::Eq, json!("user")).r#where(
"age",
ComparisonOp::Gt,
json!(35),
);
});
})
.spec();
assert_eq!(spec, expected);
}
#[test]
fn or_bracket_inside_flat_and_matches_where_then_where_any() {
let text = "role = 'user' AND (tags contains 'go' OR tags contains 'node')";
let spec = parse_query(text).unwrap();
let expected = QueryBuilder::new()
.r#where("role", ComparisonOp::Eq, json!("user"))
.where_any(|group| {
group.r#where("tags", ComparisonOp::Contains, json!("go")).r#where(
"tags",
ComparisonOp::Contains,
json!("node"),
);
})
.spec();
assert_eq!(spec, expected);
}
#[test]
fn rejects_garbage_with_a_positioned_error() {
let error = parse_query("age >=").unwrap_err();
assert_eq!(error.position, 6);
let error = parse_query("age 30").unwrap_err();
assert!(error.reason.contains("comparison operator"));
let error = parse_query("age >= 30 age").unwrap_err();
assert!(error.reason.contains("unexpected"));
let error = parse_query("(age >= 30").unwrap_err();
assert!(error.reason.contains("\")\""));
let error = parse_query("age in 30").unwrap_err();
assert!(error.reason.contains("array"));
}
#[test]
fn error_display_carries_position_and_reason() {
let error = parse_query("").unwrap_err();
let message = error.to_string();
assert!(message.contains("position"));
assert!(message.contains(&error.reason));
}
#[test]
fn order_by_defaults_to_ascending() {
let spec = parse_query("age > 30 ORDER BY age").unwrap();
let expected = QueryBuilder::new()
.r#where("age", ComparisonOp::Gt, json!(30))
.order_by("age", SortDirection::Asc)
.spec();
assert_eq!(spec, expected);
let explicit = parse_query("age > 30 ORDER BY age ASC").unwrap();
assert_eq!(explicit, expected);
}
#[test]
fn order_by_limit_offset_match_the_builder_chain() {
let text = "role = 'admin' AND age > 30 ORDER BY created_at DESC LIMIT 10 OFFSET 20";
let spec = parse_query(text).unwrap();
let expected = QueryBuilder::new()
.r#where("role", ComparisonOp::Eq, json!("admin"))
.r#where("age", ComparisonOp::Gt, json!(30))
.order_by("created_at", SortDirection::Desc)
.limit(10)
.offset(20)
.spec();
assert_eq!(spec, expected);
}
#[test]
fn clause_keywords_ignore_case_like_and_and_or() {
let upper = parse_query("age > 30 ORDER BY age DESC LIMIT 5 OFFSET 2").unwrap();
let lower = parse_query("age > 30 order by age desc limit 5 offset 2").unwrap();
let mixed = parse_query("age > 30 Order By age Desc Limit 5 Offset 2").unwrap();
let expected = QueryBuilder::new()
.r#where("age", ComparisonOp::Gt, json!(30))
.order_by("age", SortDirection::Desc)
.limit(5)
.offset(2)
.spec();
assert_eq!(upper, expected);
assert_eq!(lower, expected);
assert_eq!(mixed, expected);
}
#[test]
fn each_clause_stands_on_its_own() {
let limit_zero = parse_query("age > 30 LIMIT 0").unwrap();
assert_eq!(
limit_zero,
QueryBuilder::new()
.r#where("age", ComparisonOp::Gt, json!(30))
.limit(0)
.spec()
);
let offset_only = parse_query("age > 30 OFFSET 3").unwrap();
assert_eq!(
offset_only,
QueryBuilder::new()
.r#where("age", ComparisonOp::Gt, json!(30))
.offset(3)
.spec()
);
let bracket_then_order = parse_query("(role = 'admin' OR role = 'user') ORDER BY role").unwrap();
let expected = QueryBuilder::new()
.where_any(|group| {
group.r#where("role", ComparisonOp::Eq, json!("admin")).r#where(
"role",
ComparisonOp::Eq,
json!("user"),
);
})
.order_by("role", SortDirection::Asc)
.spec();
assert_eq!(bracket_then_order, expected);
}
#[test]
fn clauses_without_a_where_fragment_leave_the_conjunction_empty() {
let spec = parse_query("ORDER BY name LIMIT 5").unwrap();
let expected = QueryBuilder::new().order_by("name", SortDirection::Asc).limit(5).spec();
assert_eq!(spec, expected);
let limit_only = parse_query("limit 5").unwrap();
assert_eq!(limit_only, QueryBuilder::new().limit(5).spec());
let error = parse_query("").unwrap_err();
assert_eq!(error.position, 0);
assert!(error.reason.contains("field name"));
}
#[test]
fn a_field_named_like_a_clause_keyword_still_works_in_a_predicate() {
let spec = parse_query("limit > 5 AND offset in [1, 2] ORDER BY order").unwrap();
let expected = QueryBuilder::new()
.r#where("limit", ComparisonOp::Gt, json!(5))
.r#where("offset", ComparisonOp::In, json!([1, 2]))
.order_by("order", SortDirection::Asc)
.spec();
assert_eq!(spec, expected);
let leading = parse_query("order = 1 LIMIT 2").unwrap();
assert_eq!(
leading,
QueryBuilder::new()
.r#where("order", ComparisonOp::Eq, json!(1))
.limit(2)
.spec()
);
}
#[test]
fn rejects_limit_and_offset_that_are_not_non_negative_integers() {
let error = parse_query("age > 30 LIMIT -1").unwrap_err();
assert_eq!(error.position, 15);
assert_eq!(error.reason, "\"LIMIT\" needs a non-negative integer, found number -1");
let error = parse_query("age > 30 OFFSET 1.5").unwrap_err();
assert_eq!(error.position, 16);
assert_eq!(
error.reason,
"\"OFFSET\" needs a non-negative integer, found number 1.5"
);
let error = parse_query("age > 30 LIMIT 'ten'").unwrap_err();
assert_eq!(error.position, 15);
assert!(error.reason.contains("string \"ten\""));
let error = parse_query("age > 30 LIMIT").unwrap_err();
assert_eq!(error.position, 14);
assert!(error.reason.contains("end of query"));
}
#[test]
fn rejects_a_clause_given_twice() {
let error = parse_query("age > 30 LIMIT 1 LIMIT 2").unwrap_err();
assert_eq!(error.position, 17);
assert_eq!(error.reason, "\"LIMIT\" given twice — each clause may appear once");
let error = parse_query("age > 30 ORDER BY age ORDER BY name").unwrap_err();
assert_eq!(error.position, 22);
assert!(error.reason.contains("\"ORDER BY\" given twice"));
}
#[test]
fn rejects_clauses_out_of_sql_order() {
let error = parse_query("age > 30 LIMIT 5 ORDER BY age").unwrap_err();
assert_eq!(error.position, 17);
assert_eq!(error.reason, "\"ORDER BY\" must come before \"LIMIT\"");
let error = parse_query("age > 30 OFFSET 1 LIMIT 2").unwrap_err();
assert_eq!(error.position, 18);
assert_eq!(error.reason, "\"LIMIT\" must come before \"OFFSET\"");
}
#[test]
fn rejects_a_malformed_order_by() {
let error = parse_query("age > 30 ORDER age").unwrap_err();
assert_eq!(error.position, 15);
assert_eq!(
error.reason,
"expected \"BY\" after \"ORDER\", found identifier \"age\""
);
let error = parse_query("age > 30 ORDER BY").unwrap_err();
assert_eq!(error.position, 17);
assert!(error.reason.contains("field name after \"ORDER BY\""));
let error = parse_query("age > 30 ORDER BY 5").unwrap_err();
assert_eq!(error.position, 18);
assert!(error.reason.contains("found number 5"));
let error = parse_query("age > 30 ORDER BY age sideways").unwrap_err();
assert_eq!(error.position, 22);
assert!(error.reason.contains("unexpected identifier \"sideways\""));
}
}