evalhub_query/grammar.rs
1//! The `dsl-kit` grammar of a query and the parser derived from it.
2//!
3//! ```text
4//! query := { where?: filter, sort?: [sort], limit?: int, cursor?: string,
5//! expand?: [expand], version?: "latest" | "all" }
6//! filter := { and: [filter+] } | { or: [filter+] } | { not: filter } | leaf
7//! leaf := { path, op: scalar_op, value }
8//! | { path, op: "exists" }
9//! | { path, op: "any", match: { key: (value | { op, value }) ... } }
10//! scalar_op := eq | ne | gt | gte | lt | lte | in | prefix | contains
11//! sort := { path, dir: "asc" | "desc" }
12//! expand := relations | badges | fingerprints | changed
13//! ```
14//!
15//! The filter AST is declared once, as the `dsl-kit` enum [`Filter`], and
16//! from it come the conformance check, the typed builder, and the JSON
17//! Schema of the `where` value that `evalhub_schema::query` leaves open.
18//! `path` is a string at this stage; its meaning is assigned in
19//! [`crate::typecheck`].
20//!
21//! `any` with `match` is the one non-trivial construct. It applies to array
22//! paths (`results`, `relations`, `attachments`, `runs`) and matches if any
23//! element satisfies every key in `match`; a key's value is either a literal
24//! (implicit `eq`) or `{ op, value }`. This is how `results` is filtered by
25//! metric and value at once without exposing array indices.
26//!
27//! # Two shapes of vocabulary, and which one is hand-written
28//!
29//! `dsl-kit`'s JSON front end reads the internally-tagged convention
30//! (`{"type": "And", "children": […]}`), while the wire format above is
31//! key-tagged (`{"and": […]}`). [`parse`] therefore lowers the wire JSON
32//! into the tagged shape before handing it to
33//! `dsl_kit_parse::serde_bridge`, and the lowering is the only place in the
34//! workspace that names a construct by hand. It distinguishes exactly five
35//! shapes — `and`, `or`, `not`, and the two leaf forms that carry no scalar
36//! value (`exists`, `any`) — because each is a different *node shape*, and
37//! a node shape is what a `dsl-kit` variant is.
38//!
39//! The **operator vocabulary** is not hand-written anywhere: `op` is a
40//! payload field of type [`ScalarOp`], which the derived builder
41//! deserialises through `serde`. Adding `matches` or `between` to the
42//! language is one variant on that enum plus its row in the type table;
43//! no parser arm changes. That is the property the crate doc claims, and
44//! it is worth stating precisely rather than loosely, because the looser
45//! claim ("no hand-written match anywhere") is not true of the lowering.
46//!
47//! # Why the payloads are `serde_json::Value`
48//!
49//! A leaf's `value` is any JSON scalar (or an array, for `in`), and a
50//! `match` term's value likewise. `dsl-kit` payload fields convert through
51//! `serde`, so [`serde_json::Value`] is a legal field type and keeps the
52//! grammar honest about what it does *not* decide: whether `0.2` suits
53//! `generation.temperature` is a question for the type check, which has the
54//! schema. The grammar's job is shape.
55
56use std::collections::BTreeMap;
57use std::fmt;
58use std::str::FromStr;
59
60use dsl_kit_core::{IdGen, NodeId};
61use dsl_kit_macros::{DslBuild, DslNode, DslSchema};
62use dsl_kit_parse::DslBuild as _;
63use dsl_kit_parse::serde_bridge;
64use dsl_kit_schema::DslSchema as _;
65use evalhub_schema::error::{ErrorCode, ErrorEntry};
66use serde::{Deserialize, Serialize};
67use serde_json::{Map, Value, json};
68
69/// The filter AST: what a `where` clause means, before any path has been
70/// given a meaning.
71///
72/// Every variant carries the `id: NodeId` that `dsl-kit` mints, which is
73/// what lets a diagnostic point at a node rather than at a byte range.
74#[derive(Debug, Clone, PartialEq, DslNode, DslSchema, DslBuild)]
75pub enum Filter {
76 /// Every child must match. `{"and": [...]}`.
77 And {
78 /// Node identity, minted by the builder.
79 id: NodeId,
80 /// The conjuncts.
81 children: Vec<Filter>,
82 },
83 /// At least one child must match. `{"or": [...]}`.
84 Or {
85 /// Node identity, minted by the builder.
86 id: NodeId,
87 /// The disjuncts.
88 children: Vec<Filter>,
89 },
90 /// The child must not match. `{"not": {...}}`.
91 Not {
92 /// Node identity, minted by the builder.
93 id: NodeId,
94 /// The negated filter.
95 child: Box<Filter>,
96 },
97 /// A scalar comparison. `{"path": …, "op": …, "value": …}`.
98 Compare {
99 /// Node identity, minted by the builder.
100 id: NodeId,
101 /// The query path, uninterpreted here.
102 path: String,
103 /// The operator.
104 op: ScalarOp,
105 /// The literal to compare against; an array for `in`.
106 value: Value,
107 },
108 /// The path has a value, `null` included. `{"path": …, "op": "exists"}`.
109 Exists {
110 /// Node identity, minted by the builder.
111 id: NodeId,
112 /// The query path, uninterpreted here.
113 path: String,
114 },
115 /// At least one element of an array path satisfies every term.
116 /// `{"path": …, "op": "any", "match": {…}}`.
117 Any {
118 /// Node identity, minted by the builder.
119 id: NodeId,
120 /// The array path, uninterpreted here.
121 path: String,
122 /// The terms, keyed by the element field they constrain.
123 terms: BTreeMap<String, MatchTerm>,
124 },
125}
126
127/// The scalar operators, declared once.
128///
129/// This enum *is* the operator vocabulary: the builder deserialises `op`
130/// into it, [`request_schema`] lists its spellings in the published JSON
131/// Schema, and [`crate::typecheck`] decides which types each one suits.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
133#[serde(rename_all = "snake_case")]
134pub enum ScalarOp {
135 /// Equal.
136 Eq,
137 /// Not equal.
138 Ne,
139 /// Greater than.
140 Gt,
141 /// Greater than or equal.
142 Gte,
143 /// Less than.
144 Lt,
145 /// Less than or equal.
146 Lte,
147 /// One of a list.
148 In,
149 /// Starts with.
150 Prefix,
151 /// Contains.
152 Contains,
153}
154
155impl ScalarOp {
156 /// Every operator, in declaration order. Used to publish the
157 /// vocabulary without a second list.
158 pub const ALL: [ScalarOp; 9] = [
159 ScalarOp::Eq,
160 ScalarOp::Ne,
161 ScalarOp::Gt,
162 ScalarOp::Gte,
163 ScalarOp::Lt,
164 ScalarOp::Lte,
165 ScalarOp::In,
166 ScalarOp::Prefix,
167 ScalarOp::Contains,
168 ];
169
170 /// The wire spelling.
171 pub const fn as_str(self) -> &'static str {
172 match self {
173 ScalarOp::Eq => "eq",
174 ScalarOp::Ne => "ne",
175 ScalarOp::Gt => "gt",
176 ScalarOp::Gte => "gte",
177 ScalarOp::Lt => "lt",
178 ScalarOp::Lte => "lte",
179 ScalarOp::In => "in",
180 ScalarOp::Prefix => "prefix",
181 ScalarOp::Contains => "contains",
182 }
183 }
184}
185
186impl fmt::Display for ScalarOp {
187 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
188 f.write_str(self.as_str())
189 }
190}
191
192impl FromStr for ScalarOp {
193 type Err = UnknownOperator;
194
195 fn from_str(s: &str) -> Result<Self, Self::Err> {
196 ScalarOp::ALL
197 .into_iter()
198 .find(|op| op.as_str() == s)
199 .ok_or_else(|| UnknownOperator(s.to_string()))
200 }
201}
202
203/// A spelling that names no operator.
204#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
205#[error("unknown operator `{0}`")]
206pub struct UnknownOperator(pub String);
207
208/// One term of an `any` match: an operator (`eq` when the wire form was a
209/// bare literal) and the value to compare against.
210#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
211#[serde(deny_unknown_fields)]
212pub struct MatchTerm {
213 /// How to compare. Absent on the wire means `eq`.
214 #[serde(default = "MatchTerm::default_op")]
215 pub op: ScalarOp,
216 /// The literal to compare against.
217 pub value: Value,
218}
219
220impl MatchTerm {
221 const fn default_op() -> ScalarOp {
222 ScalarOp::Eq
223 }
224}
225
226impl FromStr for MatchTerm {
227 type Err = serde_json::Error;
228
229 /// Present so the term satisfies `dsl-kit`'s payload bound, which asks
230 /// for both `serde` and `FromStr` so the JSON and canonical-text front
231 /// ends cannot diverge. evalhub only uses the JSON one.
232 fn from_str(s: &str) -> Result<Self, Self::Err> {
233 serde_json::from_str(s)
234 }
235}
236
237/// The operators that change a leaf's *shape* rather than its comparison,
238/// and therefore name a `dsl-kit` variant instead of a [`ScalarOp`].
239const EXISTS_OP: &str = "exists";
240/// See [`EXISTS_OP`].
241const ANY_OP: &str = "any";
242
243/// Parse a `where` value into the filter AST.
244///
245/// Returns every structural problem it found, not just the first: a
246/// caller fixing a query should see the whole bag. Paths in the returned
247/// entries are JSON pointers into the *request* (`/where/and/1/op`), which
248/// this function tracks itself — `dsl-kit`'s JSON front end reports no
249/// position, because it reads a [`serde_json::Value`] and there is no byte
250/// offset left to report by then.
251pub fn parse(value: &Value) -> Result<Filter, Vec<ErrorEntry>> {
252 let mut errors = Vec::new();
253 let tagged = lower(value, "/where", &mut errors);
254 if !errors.is_empty() {
255 return Err(errors);
256 }
257
258 let schema = Filter::schema();
259 let tree = serde_bridge::from_json_value(&tagged, &schema)
260 .map_err(|e| diagnostics_to_entries(&e, "/where"))?;
261 Filter::from_parse_tree(&tree, &IdGen::new()).map_err(|e| diagnostics_to_entries(&e, "/where"))
262}
263
264/// Turn `dsl-kit` diagnostics into error entries. The location is always
265/// the clause as a whole, for the reason given on [`parse`]; the code is
266/// `schema`, because every one of them is a shape violation.
267fn diagnostics_to_entries(error: &dsl_kit_parse::BuildError, pointer: &str) -> Vec<ErrorEntry> {
268 error
269 .diagnostics
270 .iter()
271 .map(|d| ErrorEntry {
272 path: pointer.to_string(),
273 code: ErrorCode::Schema,
274 hint: Some(d.message.clone()),
275 })
276 .collect()
277}
278
279fn shape_error(pointer: &str, hint: impl Into<String>) -> ErrorEntry {
280 ErrorEntry {
281 path: pointer.to_string(),
282 code: ErrorCode::Schema,
283 hint: Some(hint.into()),
284 }
285}
286
287/// Lower one wire-shaped filter into `dsl-kit`'s tagged convention,
288/// recording shape problems against their JSON pointer.
289///
290/// On error the returned value is a placeholder; [`parse`] checks the
291/// error list before looking at it.
292fn lower(value: &Value, pointer: &str, errors: &mut Vec<ErrorEntry>) -> Value {
293 let Some(object) = value.as_object() else {
294 errors.push(shape_error(
295 pointer,
296 "a filter is an object: {\"and\": [...]}, {\"or\": [...]}, {\"not\": {...}} \
297 or {\"path\": ..., \"op\": ..., \"value\": ...}",
298 ));
299 return Value::Null;
300 };
301
302 // The three connectives are keyed by their name.
303 for (key, variant) in [("and", "And"), ("or", "Or")] {
304 if let Some(children) = object.get(key) {
305 if object.len() > 1 {
306 errors.push(shape_error(
307 pointer,
308 format!("`{key}` is the only key a `{key}` filter may carry"),
309 ));
310 return Value::Null;
311 }
312 let Some(items) = children.as_array() else {
313 errors.push(shape_error(
314 &format!("{pointer}/{key}"),
315 format!("`{key}` takes an array of filters"),
316 ));
317 return Value::Null;
318 };
319 let lowered: Vec<Value> = items
320 .iter()
321 .enumerate()
322 .map(|(i, child)| lower(child, &format!("{pointer}/{key}/{i}"), errors))
323 .collect();
324 return json!({ "type": variant, "children": lowered });
325 }
326 }
327 if let Some(child) = object.get("not") {
328 if object.len() > 1 {
329 errors.push(shape_error(
330 pointer,
331 "`not` is the only key a `not` filter may carry",
332 ));
333 return Value::Null;
334 }
335 let lowered = lower(child, &format!("{pointer}/not"), errors);
336 return json!({ "type": "Not", "child": lowered });
337 }
338
339 lower_leaf(object, pointer, errors)
340}
341
342fn lower_leaf(object: &Map<String, Value>, pointer: &str, errors: &mut Vec<ErrorEntry>) -> Value {
343 let Some(path) = object.get("path") else {
344 errors.push(shape_error(
345 pointer,
346 "a leaf filter needs a `path`; a connective needs `and`, `or` or `not`",
347 ));
348 return Value::Null;
349 };
350 let Some(path) = path.as_str() else {
351 errors.push(shape_error(
352 &format!("{pointer}/path"),
353 "`path` is a string",
354 ));
355 return Value::Null;
356 };
357 let Some(op) = object.get("op") else {
358 errors.push(shape_error(
359 pointer,
360 format!(
361 "a leaf filter needs an `op`: one of {}, `{EXISTS_OP}` or `{ANY_OP}`",
362 operator_list()
363 ),
364 ));
365 return Value::Null;
366 };
367 let Some(op) = op.as_str() else {
368 errors.push(shape_error(&format!("{pointer}/op"), "`op` is a string"));
369 return Value::Null;
370 };
371
372 match op {
373 EXISTS_OP => {
374 reject_extra_keys(object, &["path", "op"], pointer, errors);
375 json!({ "type": "Exists", "path": path })
376 }
377 ANY_OP => {
378 reject_extra_keys(object, &["path", "op", "match"], pointer, errors);
379 let Some(terms) = object.get("match") else {
380 errors.push(shape_error(
381 pointer,
382 "`any` needs a `match` object naming the element fields to constrain",
383 ));
384 return Value::Null;
385 };
386 let Some(terms) = terms.as_object() else {
387 errors.push(shape_error(
388 &format!("{pointer}/match"),
389 "`match` is an object",
390 ));
391 return Value::Null;
392 };
393 let lowered: Map<String, Value> = terms
394 .iter()
395 .map(|(key, term)| {
396 (
397 key.clone(),
398 lower_term(term, &format!("{pointer}/match/{key}"), errors),
399 )
400 })
401 .collect();
402 json!({ "type": "Any", "path": path, "terms": lowered })
403 }
404 _ => {
405 reject_extra_keys(object, &["path", "op", "value"], pointer, errors);
406 let Some(value) = object.get("value") else {
407 errors.push(shape_error(
408 pointer,
409 format!("`{op}` needs a `value`; only `{EXISTS_OP}` stands alone"),
410 ));
411 return Value::Null;
412 };
413 json!({ "type": "Compare", "path": path, "op": op, "value": value })
414 }
415 }
416}
417
418/// A `match` term is either a bare literal (meaning `eq`) or
419/// `{ op, value }`. Both are lowered to the explicit form, which is what
420/// [`MatchTerm`] deserialises from; `dsl-kit` hands each entry of a keyed
421/// scalar slot to `serde` verbatim.
422fn lower_term(term: &Value, pointer: &str, errors: &mut Vec<ErrorEntry>) -> Value {
423 match term.as_object() {
424 Some(object) if object.contains_key("op") || object.contains_key("value") => {
425 reject_extra_keys(object, &["op", "value"], pointer, errors);
426 if !object.contains_key("value") {
427 errors.push(shape_error(
428 pointer,
429 "a match term with an `op` also needs a `value`",
430 ));
431 return Value::Null;
432 }
433 json!({
434 "op": object.get("op").cloned().unwrap_or_else(|| json!("eq")),
435 "value": object.get("value"),
436 })
437 }
438 // Any other value — including an object that is plainly a literal —
439 // is the implicit-`eq` form.
440 _ => json!({ "op": "eq", "value": term }),
441 }
442}
443
444fn reject_extra_keys(
445 object: &Map<String, Value>,
446 allowed: &[&str],
447 pointer: &str,
448 errors: &mut Vec<ErrorEntry>,
449) {
450 for key in object.keys() {
451 if !allowed.contains(&key.as_str()) {
452 errors.push(shape_error(
453 &format!("{pointer}/{key}"),
454 format!("unknown key `{key}`; this filter takes {}", list(allowed)),
455 ));
456 }
457 }
458}
459
460fn list(items: &[&str]) -> String {
461 items
462 .iter()
463 .map(|i| format!("`{i}`"))
464 .collect::<Vec<_>>()
465 .join(", ")
466}
467
468fn operator_list() -> String {
469 ScalarOp::ALL
470 .iter()
471 .map(|op| format!("`{op}`"))
472 .collect::<Vec<_>>()
473 .join(", ")
474}
475
476/// The wire key a connective is spelled with, so that the type check can
477/// rebuild the JSON pointer of a nested filter without a second table of
478/// shape names. Leaves have no key of their own: they sit where their
479/// parent put them.
480pub(crate) fn wire_key(filter: &Filter) -> &'static str {
481 match filter {
482 Filter::And { .. } => "and",
483 Filter::Or { .. } => "or",
484 Filter::Not { .. } => "not",
485 Filter::Compare { .. } | Filter::Exists { .. } | Filter::Any { .. } => "",
486 }
487}
488
489/// The JSON Schema of a `where` value, for the OpenAPI document.
490///
491/// `evalhub_schema::query::QueryRequest` leaves `where` open because the
492/// filter's shape belongs to this crate; the server splices this schema in
493/// so a generated client sees the real thing.
494///
495/// The schema describes the *wire* shape, which is the inverse of the
496/// lowering in [`parse`]: `dsl-kit`'s [`dsl_kit_schema::NodeSchema`]
497/// describes the tagged shape, so it is read for the variant and field
498/// names rather than copied. What it cannot express is carried here
499/// instead: which keys pair with which operator. The operator list comes
500/// from [`ScalarOp::ALL`], so the published vocabulary cannot drift from
501/// the one the parser accepts.
502pub fn request_schema() -> schemars::Schema {
503 let node = Filter::schema();
504 // Read the leaf field names off the declared schema rather than
505 // spelling them again; if a variant is renamed the schema follows.
506 let compare = node
507 .variant("Compare")
508 .map(|v| v.fields.iter().map(|f| f.name.clone()).collect::<Vec<_>>())
509 .unwrap_or_default();
510 let path_key = compare
511 .iter()
512 .find(|f| *f == "path")
513 .cloned()
514 .unwrap_or_else(|| "path".to_string());
515 let value_key = compare
516 .iter()
517 .find(|f| *f == "value")
518 .cloned()
519 .unwrap_or_else(|| "value".to_string());
520 let ops: Vec<Value> = ScalarOp::ALL.iter().map(|op| json!(op.as_str())).collect();
521
522 let filter_ref = json!({ "$ref": "#/$defs/filter" });
523 let term = json!({
524 "description": "A literal (meaning `eq`), or an operator and a literal.",
525 "oneOf": [
526 { "type": ["string", "number", "boolean", "null"] },
527 {
528 "type": "object",
529 "properties": {
530 "op": { "enum": ops.clone() },
531 &value_key: true,
532 },
533 "required": [&value_key],
534 "additionalProperties": false,
535 },
536 ],
537 });
538
539 let document = json!({
540 "$ref": "#/$defs/filter",
541 "$defs": {
542 "filter": {
543 "description": "A filter: a connective or a leaf.",
544 "oneOf": [
545 {
546 "type": "object",
547 "properties": { "and": { "type": "array", "items": filter_ref } },
548 "required": ["and"],
549 "additionalProperties": false,
550 },
551 {
552 "type": "object",
553 "properties": { "or": { "type": "array", "items": filter_ref } },
554 "required": ["or"],
555 "additionalProperties": false,
556 },
557 {
558 "type": "object",
559 "properties": { "not": filter_ref },
560 "required": ["not"],
561 "additionalProperties": false,
562 },
563 {
564 "description": "A scalar comparison.",
565 "type": "object",
566 "properties": {
567 &path_key: { "type": "string" },
568 "op": { "enum": ops },
569 &value_key: true,
570 },
571 "required": [&path_key, "op", &value_key],
572 "additionalProperties": false,
573 },
574 {
575 "description": "The path has a value, `null` included.",
576 "type": "object",
577 "properties": {
578 &path_key: { "type": "string" },
579 "op": { "const": EXISTS_OP },
580 },
581 "required": [&path_key, "op"],
582 "additionalProperties": false,
583 },
584 {
585 "description": "At least one element of an array path satisfies every term.",
586 "type": "object",
587 "properties": {
588 &path_key: { "type": "string" },
589 "op": { "const": ANY_OP },
590 "match": { "type": "object", "additionalProperties": term },
591 },
592 "required": [&path_key, "op", "match"],
593 "additionalProperties": false,
594 },
595 ],
596 },
597 },
598 });
599 // `json!` of an object literal is always an object, and schemars builds
600 // a schema from one directly.
601 match document {
602 Value::Object(map) => schemars::Schema::from(map),
603 _ => schemars::Schema::default(),
604 }
605}