evalhub_query/ir.rs
1//! The backend-independent query tree.
2//!
3//! [`Query`] is what the type check produces and what
4//! `evalhub_store::query_sql` consumes. Every path in it has been resolved
5//! to a [`Column`] that names *where* the value lives (a generated column, a
6//! fingerprint row, a side table, a JSON expression) so the SQL compiler
7//! needs no knowledge of the schema — it only needs to know how to render
8//! each `Column` kind. That split keeps the SQL compiler small and keeps
9//! this crate free of SQL.
10//!
11//! ```text
12//! Query { record_type, filter, sort, limit, cursor, expand, version }
13//! RunQuery { cards, filter, sort, limit, cursor: RunCursor }
14//! Filter := And(Vec) | Or(Vec) | Not(Box) | Cmp { column, op, value }
15//! | Exists { column } | Any { table, conditions: Vec<Cmp> }
16//! Column := Generated(name) | Fingerprint(facet) | Ext { path, ty } | Body(json_path)
17//! | Array(column) | Run(RunColumn)
18//! RunColumn := RunId | Status | ErrorKind | StartedAt | EndedAt | Facet(json_path)
19//! | Fingerprint(facet) | Metric(id) | Result { card, metric, field }
20//! Cursor := { keys: Vec<Value>, version_id } // decoded; encoding is the server's
21//! RunCursor := { keys: Vec<Value>, run_id } // the same, for the run projection
22//! ```
23//!
24//! # Two targets
25//!
26//! [`Query`] searches records (`POST /cards/query`, `POST /evals/query`):
27//! one row per version. [`RunQuery`] reads the run projection of one Eval
28//! (`GET /evals/{ns}/{name}/runs`): one row per run of that Eval, joined
29//! with the `run_results` of the Cards the request names. The filter tree
30//! is shared — the grammar does not change between them — but the columns
31//! a run query resolves to are [`Column::Run`], plus [`Column::Ext`] and
32//! [`Column::Body`] for the run's own `ext`, which in a [`RunQuery`] are
33//! paths below the *run's* body, not a version's. A record query never
34//! contains [`Column::Run`], and a run query never contains
35//! [`Column::Generated`], [`Column::Fingerprint`] or [`Column::Array`];
36//! a backend may refuse either stray as unsupported.
37//!
38//! The run projection has its own cursor, [`RunCursor`], because the row
39//! it pages over is keyed by `run_id` (unique within the Eval), not by a
40//! `version_id`.
41//!
42//! The IR is `serde`-serialisable so a query can be snapshot-tested end to
43//! end: JSON in, IR out, without a database.
44//!
45//! # Why the column kinds are closed
46//!
47//! Each variant of [`Column`] corresponds to one storage decision made in
48//! the migration: a generated column, the `fingerprints` table, an
49//! expression index over a registered `ext` path, or the GIN index over the
50//! whole body. A path the type check cannot place in one of them is not a
51//! query the hub can serve, and saying so (`422 not_indexed`) is better
52//! than a sequential scan that looks like a feature until the table grows.
53//!
54//! # Values
55//!
56//! Literals travel as `serde_json::Value` and are bound as parameters, never
57//! rendered into SQL text. The type check has already confirmed that a
58//! literal matches its column's type, so the compiler binds by
59//! [`ValueType`] rather than by inspecting the JSON again.
60
61use serde::{Deserialize, Serialize};
62use serde_json::Value;
63
64/// A checked, resolved query ready for a backend to compile.
65#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
66pub struct Query {
67 /// Which record kind is being searched. The compiler adds
68 /// `records.type = …`.
69 pub record_type: RecordType,
70 /// The predicate. `None` matches every visible record.
71 pub filter: Option<Filter>,
72 /// Sort keys, most significant first. Empty means the backend's
73 /// default order (newest version first).
74 pub sort: Vec<Sort>,
75 /// Page size the caller asked for, already clamped by the server.
76 pub limit: u32,
77 /// Where the previous page stopped, decoded and verified by the server.
78 pub cursor: Option<Cursor>,
79 /// What to include alongside each record.
80 pub expand: Vec<Expand>,
81 /// Whether to consider every version or only the latest live one.
82 pub version: VersionSelector,
83}
84
85/// The record kind a query searches.
86#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
87#[serde(rename_all = "snake_case")]
88pub enum RecordType {
89 /// Cards: what was measured, how, and what the score was.
90 Card,
91 /// Evals: the material a Card was measured from.
92 Eval,
93}
94
95/// Which versions a query considers.
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
97#[serde(rename_all = "snake_case")]
98pub enum VersionSelector {
99 /// Only the highest non-tombstoned `seq` of each name.
100 #[default]
101 Latest,
102 /// Every non-tombstoned version.
103 All,
104}
105
106/// What to include alongside each matched record.
107#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
108#[serde(rename_all = "snake_case")]
109pub enum Expand {
110 /// The version's outgoing relations.
111 Relations,
112 /// The badges awarded at ingest.
113 Badges,
114 /// The seven per-facet fingerprints.
115 Fingerprints,
116 /// The top-level keys that differ from the previous version.
117 Changed,
118}
119
120/// The predicate tree. `And`/`Or` with no children are the identity of
121/// their operation, which is what an empty `{"and": []}` means.
122#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
123#[serde(rename_all = "snake_case")]
124pub enum Filter {
125 /// Every child must match.
126 And(Vec<Filter>),
127 /// At least one child must match.
128 Or(Vec<Filter>),
129 /// The child must not match.
130 Not(Box<Filter>),
131 /// A scalar comparison against one column.
132 Cmp(Cmp),
133 /// The column has a value, `null` included.
134 Exists {
135 /// Where the value lives.
136 column: Column,
137 },
138 /// At least one row of a side table satisfies every condition.
139 Any {
140 /// The table to look in.
141 table: ArrayTable,
142 /// Conditions on that table's columns; all must hold of one row.
143 conditions: Vec<Cmp>,
144 },
145}
146
147/// One scalar comparison.
148#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
149pub struct Cmp {
150 /// Where the value lives.
151 pub column: Column,
152 /// How to compare.
153 pub op: Op,
154 /// What to compare against. `In` carries an array; every other
155 /// operator carries a scalar.
156 pub value: Value,
157}
158
159/// The comparison operators, after the type check has confirmed each one
160/// suits its column.
161#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
162#[serde(rename_all = "snake_case")]
163pub enum Op {
164 /// Equal.
165 Eq,
166 /// Not equal.
167 Ne,
168 /// Greater than.
169 Gt,
170 /// Greater than or equal.
171 Gte,
172 /// Less than.
173 Lt,
174 /// Less than or equal.
175 Lte,
176 /// One of a list.
177 In,
178 /// Starts with (strings).
179 Prefix,
180 /// Contains (strings).
181 Contains,
182}
183
184/// A side table an `any` condition ranges over. Each has its own columns,
185/// listed in [`ArrayColumn`].
186#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
187#[serde(rename_all = "snake_case")]
188pub enum ArrayTable {
189 /// `results (metric, aggregation, value, n)`.
190 Results,
191 /// `relations (type, to_version_id, to_external)`.
192 Relations,
193 /// `attachment_refs (path, sha256)`.
194 Attachments,
195}
196
197/// Where a value lives, in terms the storage layer can render directly.
198#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
199#[serde(rename_all = "snake_case")]
200pub enum Column {
201 /// A generated column on `versions`, named by the migration.
202 Generated(String),
203 /// A facet's fingerprint, matched against the `fingerprints` table.
204 Fingerprint(String),
205 /// A registered `ext` path, served by an expression index. The
206 /// compiler must render the *same* expression text the index was
207 /// created with, which is why the path and the type travel together.
208 /// In a [`RunQuery`] the path is below the run's body, not a
209 /// version's.
210 Ext {
211 /// JSON path segments below the document root, `ext` included.
212 path: Vec<String>,
213 /// The registered type, which picks the cast helper.
214 ty: ValueType,
215 },
216 /// Any other path, served by the GIN index over the whole body.
217 /// Only equality and existence are allowed here, which is what that
218 /// index answers. In a [`RunQuery`] the path is below the run's body
219 /// (an unregistered `ext` key of the run).
220 Body(Vec<String>),
221 /// A column of the side table an `any` condition ranges over.
222 Array(ArrayColumn),
223 /// A column of the run projection. Only a [`RunQuery`] carries these.
224 Run(RunColumn),
225}
226
227/// Where a value of the run projection lives. One variant per storage
228/// decision of the run tables: a column of `runs`, the run's body, the
229/// `run_fingerprints` / `run_metrics` side tables, or a Card's
230/// `run_results` joined by `run_id`.
231///
232/// Every column is scoped to the runs of one Eval: the projection is read
233/// through `GET /evals/{ns}/{name}/runs`, never across Evals.
234#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
235#[serde(rename_all = "snake_case")]
236pub enum RunColumn {
237 /// `runs.run_id`. A string, unique within the Eval.
238 RunId,
239 /// `runs.status`: `ok`, `error` or `skipped`.
240 Status,
241 /// `runs.error_kind`, the producer's slug; null unless `status` is
242 /// `error`.
243 ErrorKind,
244 /// `runs.started_at`. The DSL types it as a string (RFC 3339); a
245 /// backend compares it as a timestamp.
246 StartedAt,
247 /// `runs.ended_at`, as [`RunColumn::StartedAt`].
248 EndedAt,
249 /// A key of one of the run's six facets, as JSON path segments below
250 /// the run's body (`["model", "id"]`). The run stores its facets after
251 /// the header's defaults were copied onto it, so this is the run's
252 /// own condition, never the header's.
253 Facet(Vec<String>),
254 /// The fingerprint of one of the run's facets (`run_fingerprints`),
255 /// hex-encoded as the API returns it.
256 Fingerprint(String),
257 /// One of the run's own `metrics`, by registry id `{ns}/{name}`
258 /// (`run_metrics`). A run without that metric has no value: a
259 /// comparison is false for it, and it sorts as the backend sorts nulls.
260 Metric(String),
261 /// A judgement from one of the Cards the request names: that Card's
262 /// latest live version's `run_results` entry for this run and
263 /// `metric`. A run the Card did not judge on that metric has no value.
264 Result {
265 /// The Card, as named in the request.
266 card: CardRef,
267 /// The `run_results[].metric`, `{ns}/{name}`.
268 metric: String,
269 /// Which field of the entry.
270 field: ResultField,
271 },
272}
273
274/// Which field of a Card's `run_results` entry a [`RunColumn::Result`]
275/// reads.
276#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
277#[serde(rename_all = "snake_case")]
278pub enum ResultField {
279 /// `run_results[].value`, a number.
280 Value,
281 /// `run_results[].label`, a string.
282 Label,
283}
284
285/// A Card named by a run projection request: `{ns}/{name}`, joined at its
286/// latest live version.
287///
288/// `GET /evals/{ns}/{name}/runs?cards=…` names the Cards whose judgements
289/// the projection carries, and each is joined at the version a reader
290/// would get from `GET /cards/{ns}/{name}`. There is no `@seq` form: the
291/// projection answers "what do these graders say about these runs now",
292/// and a Card's earlier versions remain readable on their own. Whether the
293/// caller may see each Card is the server's decision, made before the
294/// path table is built; this crate only knows the names.
295///
296/// Both parts follow the registry id rule (`[a-z0-9][a-z0-9._-]*`), so a
297/// Card reference never contains `[`, `]` or a second `/`, which is what
298/// lets `results[{card}][{metric}]` be split without escaping.
299#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
300#[serde(try_from = "String", into = "String")]
301pub struct CardRef(String);
302
303impl CardRef {
304 /// Parse `{ns}/{name}`, or `None` when it is not of that form.
305 pub fn parse(s: &str) -> Option<Self> {
306 evalhub_core::validate::is_valid_id(s).then(|| Self(s.to_string()))
307 }
308
309 /// The text form, `{ns}/{name}`.
310 pub fn as_str(&self) -> &str {
311 &self.0
312 }
313
314 /// The namespace part.
315 pub fn ns(&self) -> &str {
316 self.0.split_once('/').map_or("", |(ns, _)| ns)
317 }
318
319 /// The name part.
320 pub fn name(&self) -> &str {
321 self.0.split_once('/').map_or("", |(_, name)| name)
322 }
323}
324
325impl std::fmt::Display for CardRef {
326 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
327 f.write_str(&self.0)
328 }
329}
330
331impl std::str::FromStr for CardRef {
332 type Err = InvalidCardRef;
333 fn from_str(s: &str) -> Result<Self, Self::Err> {
334 Self::parse(s).ok_or_else(|| InvalidCardRef(s.to_string()))
335 }
336}
337
338impl TryFrom<String> for CardRef {
339 type Error = InvalidCardRef;
340 fn try_from(s: String) -> Result<Self, Self::Error> {
341 s.parse()
342 }
343}
344
345impl From<CardRef> for String {
346 fn from(card: CardRef) -> Self {
347 card.0
348 }
349}
350
351/// The text was not a Card reference of the form `{ns}/{name}`.
352#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
353#[error("not a Card reference of the form {{ns}}/{{name}}: {0:?}")]
354pub struct InvalidCardRef(pub String);
355
356/// The columns an `any` condition may compare, one set per table.
357#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
358#[serde(rename_all = "snake_case")]
359pub enum ArrayColumn {
360 /// `results.metric`
361 ResultMetric,
362 /// `results.aggregation`
363 ResultAggregation,
364 /// `results.value`
365 ResultValue,
366 /// `results.n`
367 ResultN,
368 /// `relations.type`
369 RelationType,
370 /// The relation's target, resolved or textual.
371 RelationTo,
372 /// `attachment_refs.path`
373 AttachmentPath,
374 /// `attachment_refs.sha256`
375 AttachmentSha256,
376}
377
378impl ArrayColumn {
379 /// The table this column belongs to, so a compiler can reject a
380 /// condition that strayed into another one.
381 pub fn table(self) -> ArrayTable {
382 match self {
383 Self::ResultMetric | Self::ResultAggregation | Self::ResultValue | Self::ResultN => {
384 ArrayTable::Results
385 }
386 Self::RelationType | Self::RelationTo => ArrayTable::Relations,
387 Self::AttachmentPath | Self::AttachmentSha256 => ArrayTable::Attachments,
388 }
389 }
390}
391
392/// The scalar types the query language distinguishes. They decide the cast
393/// helper for an `ext` path and the binding a literal gets.
394#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
395#[serde(rename_all = "snake_case")]
396pub enum ValueType {
397 /// IEEE 754 double, as every JSON number is.
398 Number,
399 /// UTF-8 text.
400 String,
401 /// `true` or `false`.
402 Boolean,
403}
404
405/// One sort key.
406#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
407pub struct Sort {
408 /// What to sort by.
409 pub key: SortKey,
410 /// Which way.
411 pub dir: Dir,
412}
413
414/// What a sort orders by.
415#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
416#[serde(rename_all = "snake_case")]
417pub enum SortKey {
418 /// A column of `versions`, or a registered `ext` path.
419 Column(Column),
420 /// The value of one metric, which is a lateral join on `results`.
421 Metric(String),
422 /// When the version was stored.
423 CreatedAt,
424 /// A column of the run projection. Only a [`RunQuery`] carries these;
425 /// a run query sorts on a registered `ext` path of the run with
426 /// [`SortKey::Column`] and [`Column::Ext`].
427 Run(RunColumn),
428}
429
430/// Sort direction.
431#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
432#[serde(rename_all = "snake_case")]
433pub enum Dir {
434 /// Smallest first.
435 Asc,
436 /// Largest first.
437 Desc,
438}
439
440/// Where the previous page stopped: the sort keys of its last row and that
441/// row's `version_id`, which breaks ties and makes the tuple unique.
442///
443/// The server signs and encodes this; the store only compares against it.
444#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
445pub struct Cursor {
446 /// One value per entry of [`Query::sort`], in the same order.
447 pub keys: Vec<Value>,
448 /// The last row's version id.
449 pub version_id: uuid::Uuid,
450}
451
452/// A checked, resolved read of one Eval's run projection, ready for a
453/// backend to compile.
454///
455/// Which Eval, and whether archived and deleted runs are included
456/// (`include=archived,deleted`, members only), are the server's to decide
457/// and pass beside this; the DSL knows neither. With no sort the backend
458/// orders by `run_id` ascending.
459#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
460pub struct RunQuery {
461 /// The Cards whose `run_results` are joined, in the order the request
462 /// named them. Every [`RunColumn::Result`] in the filter and the sort
463 /// names one of these; the projection carries all of them whether or
464 /// not a condition mentions them.
465 pub cards: Vec<CardRef>,
466 /// The predicate. `None` matches every run the server lets through.
467 pub filter: Option<Filter>,
468 /// Sort keys, most significant first. `run_id` is always the final,
469 /// implicit tie-breaker.
470 pub sort: Vec<Sort>,
471 /// Page size the caller asked for, already clamped by the server.
472 pub limit: u32,
473 /// Where the previous page stopped, decoded and verified by the server.
474 pub cursor: Option<RunCursor>,
475}
476
477/// Where the previous page of a run projection stopped: the sort keys of
478/// its last row and that row's `run_id`, which breaks ties and makes the
479/// tuple unique within the Eval.
480///
481/// [`Cursor`] cannot serve here, because its tie-breaker is a
482/// `version_id` and a run is not a version. The server treats this exactly
483/// as it treats a [`Cursor`]: serialises it as JSON, signs the bytes, and
484/// hands the client an opaque string; on the next request it verifies the
485/// signature and deserialises. The store only compares against it.
486#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
487pub struct RunCursor {
488 /// One value per entry of [`RunQuery::sort`], in the same order. A run
489 /// with no value for a key (a metric it lacks) carries `null`.
490 pub keys: Vec<Value>,
491 /// The last row's `run_id`.
492 pub run_id: String,
493}