Skip to main content

evalhub_query/
ir.rs

1//! The backend-independent query tree.
2//!
3//! [`Query`] is what the type check produces and what
4//! `evalhub_store::query_sql` consumes. Every path in it has been resolved
5//! to a [`Column`] that names *where* the value lives (a generated column, a
6//! fingerprint row, a side table, a JSON expression) so the SQL compiler
7//! needs no knowledge of the schema — it only needs to know how to render
8//! each `Column` kind. That split keeps the SQL compiler small and keeps
9//! this crate free of SQL.
10//!
11//! ```text
12//! Query    { record_type, filter, sort, limit, cursor, expand, version }
13//! RunQuery { cards, filter, sort, limit, cursor: RunCursor }
14//! Filter := And(Vec) | Or(Vec) | Not(Box) | Cmp { column, op, value }
15//!         | Exists { column } | Any { table, conditions: Vec<Cmp> }
16//! Column := Generated(name) | Fingerprint(facet) | Ext { path, ty } | Body(json_path)
17//!         | Array(column) | Run(RunColumn)
18//! RunColumn := RunId | Status | ErrorKind | StartedAt | EndedAt | Facet(json_path)
19//!            | Fingerprint(facet) | Metric(id) | Result { card, metric, field }
20//! Cursor    := { keys: Vec<Value>, version_id }   // decoded; encoding is the server's
21//! RunCursor := { keys: Vec<Value>, run_id }       // the same, for the run projection
22//! ```
23//!
24//! # Two targets
25//!
26//! [`Query`] searches records (`POST /cards/query`, `POST /evals/query`):
27//! one row per version. [`RunQuery`] reads the run projection of one Eval
28//! (`GET /evals/{ns}/{name}/runs`): one row per run of that Eval, joined
29//! with the `run_results` of the Cards the request names. The filter tree
30//! is shared — the grammar does not change between them — but the columns
31//! a run query resolves to are [`Column::Run`], plus [`Column::Ext`] and
32//! [`Column::Body`] for the run's own `ext`, which in a [`RunQuery`] are
33//! paths below the *run's* body, not a version's. A record query never
34//! contains [`Column::Run`], and a run query never contains
35//! [`Column::Generated`], [`Column::Fingerprint`] or [`Column::Array`];
36//! a backend may refuse either stray as unsupported.
37//!
38//! The run projection has its own cursor, [`RunCursor`], because the row
39//! it pages over is keyed by `run_id` (unique within the Eval), not by a
40//! `version_id`.
41//!
42//! The IR is `serde`-serialisable so a query can be snapshot-tested end to
43//! end: JSON in, IR out, without a database.
44//!
45//! # Why the column kinds are closed
46//!
47//! Each variant of [`Column`] corresponds to one storage decision made in
48//! the migration: a generated column, the `fingerprints` table, an
49//! expression index over a registered `ext` path, or the GIN index over the
50//! whole body. A path the type check cannot place in one of them is not a
51//! query the hub can serve, and saying so (`422 not_indexed`) is better
52//! than a sequential scan that looks like a feature until the table grows.
53//!
54//! # Values
55//!
56//! Literals travel as `serde_json::Value` and are bound as parameters, never
57//! rendered into SQL text. The type check has already confirmed that a
58//! literal matches its column's type, so the compiler binds by
59//! [`ValueType`] rather than by inspecting the JSON again.
60
61use serde::{Deserialize, Serialize};
62use serde_json::Value;
63
64/// A checked, resolved query ready for a backend to compile.
65#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
66pub struct Query {
67    /// Which record kind is being searched. The compiler adds
68    /// `records.type = …`.
69    pub record_type: RecordType,
70    /// The predicate. `None` matches every visible record.
71    pub filter: Option<Filter>,
72    /// Sort keys, most significant first. Empty means the backend's
73    /// default order (newest version first).
74    pub sort: Vec<Sort>,
75    /// Page size the caller asked for, already clamped by the server.
76    pub limit: u32,
77    /// Where the previous page stopped, decoded and verified by the server.
78    pub cursor: Option<Cursor>,
79    /// What to include alongside each record.
80    pub expand: Vec<Expand>,
81    /// Whether to consider every version or only the latest live one.
82    pub version: VersionSelector,
83}
84
85/// The record kind a query searches.
86#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
87#[serde(rename_all = "snake_case")]
88pub enum RecordType {
89    /// Cards: what was measured, how, and what the score was.
90    Card,
91    /// Evals: the material a Card was measured from.
92    Eval,
93}
94
95/// Which versions a query considers.
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
97#[serde(rename_all = "snake_case")]
98pub enum VersionSelector {
99    /// Only the highest non-tombstoned `seq` of each name.
100    #[default]
101    Latest,
102    /// Every non-tombstoned version.
103    All,
104}
105
106/// What to include alongside each matched record.
107#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
108#[serde(rename_all = "snake_case")]
109pub enum Expand {
110    /// The version's outgoing relations.
111    Relations,
112    /// The badges awarded at ingest.
113    Badges,
114    /// The seven per-facet fingerprints.
115    Fingerprints,
116    /// The top-level keys that differ from the previous version.
117    Changed,
118}
119
120/// The predicate tree. `And`/`Or` with no children are the identity of
121/// their operation, which is what an empty `{"and": []}` means.
122#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
123#[serde(rename_all = "snake_case")]
124pub enum Filter {
125    /// Every child must match.
126    And(Vec<Filter>),
127    /// At least one child must match.
128    Or(Vec<Filter>),
129    /// The child must not match.
130    Not(Box<Filter>),
131    /// A scalar comparison against one column.
132    Cmp(Cmp),
133    /// The column has a value, `null` included.
134    Exists {
135        /// Where the value lives.
136        column: Column,
137    },
138    /// At least one row of a side table satisfies every condition.
139    Any {
140        /// The table to look in.
141        table: ArrayTable,
142        /// Conditions on that table's columns; all must hold of one row.
143        conditions: Vec<Cmp>,
144    },
145}
146
147/// One scalar comparison.
148#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
149pub struct Cmp {
150    /// Where the value lives.
151    pub column: Column,
152    /// How to compare.
153    pub op: Op,
154    /// What to compare against. `In` carries an array; every other
155    /// operator carries a scalar.
156    pub value: Value,
157}
158
159/// The comparison operators, after the type check has confirmed each one
160/// suits its column.
161#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
162#[serde(rename_all = "snake_case")]
163pub enum Op {
164    /// Equal.
165    Eq,
166    /// Not equal.
167    Ne,
168    /// Greater than.
169    Gt,
170    /// Greater than or equal.
171    Gte,
172    /// Less than.
173    Lt,
174    /// Less than or equal.
175    Lte,
176    /// One of a list.
177    In,
178    /// Starts with (strings).
179    Prefix,
180    /// Contains (strings).
181    Contains,
182}
183
184/// A side table an `any` condition ranges over. Each has its own columns,
185/// listed in [`ArrayColumn`].
186#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
187#[serde(rename_all = "snake_case")]
188pub enum ArrayTable {
189    /// `results (metric, aggregation, value, n)`.
190    Results,
191    /// `relations (type, to_version_id, to_external)`.
192    Relations,
193    /// `attachment_refs (path, sha256)`.
194    Attachments,
195}
196
197/// Where a value lives, in terms the storage layer can render directly.
198#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
199#[serde(rename_all = "snake_case")]
200pub enum Column {
201    /// A generated column on `versions`, named by the migration.
202    Generated(String),
203    /// A facet's fingerprint, matched against the `fingerprints` table.
204    Fingerprint(String),
205    /// A registered `ext` path, served by an expression index. The
206    /// compiler must render the *same* expression text the index was
207    /// created with, which is why the path and the type travel together.
208    /// In a [`RunQuery`] the path is below the run's body, not a
209    /// version's.
210    Ext {
211        /// JSON path segments below the document root, `ext` included.
212        path: Vec<String>,
213        /// The registered type, which picks the cast helper.
214        ty: ValueType,
215    },
216    /// Any other path, served by the GIN index over the whole body.
217    /// Only equality and existence are allowed here, which is what that
218    /// index answers. In a [`RunQuery`] the path is below the run's body
219    /// (an unregistered `ext` key of the run).
220    Body(Vec<String>),
221    /// A column of the side table an `any` condition ranges over.
222    Array(ArrayColumn),
223    /// A column of the run projection. Only a [`RunQuery`] carries these.
224    Run(RunColumn),
225}
226
227/// Where a value of the run projection lives. One variant per storage
228/// decision of the run tables: a column of `runs`, the run's body, the
229/// `run_fingerprints` / `run_metrics` side tables, or a Card's
230/// `run_results` joined by `run_id`.
231///
232/// Every column is scoped to the runs of one Eval: the projection is read
233/// through `GET /evals/{ns}/{name}/runs`, never across Evals.
234#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
235#[serde(rename_all = "snake_case")]
236pub enum RunColumn {
237    /// `runs.run_id`. A string, unique within the Eval.
238    RunId,
239    /// `runs.status`: `ok`, `error` or `skipped`.
240    Status,
241    /// `runs.error_kind`, the producer's slug; null unless `status` is
242    /// `error`.
243    ErrorKind,
244    /// `runs.started_at`. The DSL types it as a string (RFC 3339); a
245    /// backend compares it as a timestamp.
246    StartedAt,
247    /// `runs.ended_at`, as [`RunColumn::StartedAt`].
248    EndedAt,
249    /// A key of one of the run's six facets, as JSON path segments below
250    /// the run's body (`["model", "id"]`). The run stores its facets after
251    /// the header's defaults were copied onto it, so this is the run's
252    /// own condition, never the header's.
253    Facet(Vec<String>),
254    /// The fingerprint of one of the run's facets (`run_fingerprints`),
255    /// hex-encoded as the API returns it.
256    Fingerprint(String),
257    /// One of the run's own `metrics`, by registry id `{ns}/{name}`
258    /// (`run_metrics`). A run without that metric has no value: a
259    /// comparison is false for it, and it sorts as the backend sorts nulls.
260    Metric(String),
261    /// A judgement from one of the Cards the request names: that Card's
262    /// latest live version's `run_results` entry for this run and
263    /// `metric`. A run the Card did not judge on that metric has no value.
264    Result {
265        /// The Card, as named in the request.
266        card: CardRef,
267        /// The `run_results[].metric`, `{ns}/{name}`.
268        metric: String,
269        /// Which field of the entry.
270        field: ResultField,
271    },
272}
273
274/// Which field of a Card's `run_results` entry a [`RunColumn::Result`]
275/// reads.
276#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
277#[serde(rename_all = "snake_case")]
278pub enum ResultField {
279    /// `run_results[].value`, a number.
280    Value,
281    /// `run_results[].label`, a string.
282    Label,
283}
284
285/// A Card named by a run projection request: `{ns}/{name}`, joined at its
286/// latest live version.
287///
288/// `GET /evals/{ns}/{name}/runs?cards=…` names the Cards whose judgements
289/// the projection carries, and each is joined at the version a reader
290/// would get from `GET /cards/{ns}/{name}`. There is no `@seq` form: the
291/// projection answers "what do these graders say about these runs now",
292/// and a Card's earlier versions remain readable on their own. Whether the
293/// caller may see each Card is the server's decision, made before the
294/// path table is built; this crate only knows the names.
295///
296/// Both parts follow the registry id rule (`[a-z0-9][a-z0-9._-]*`), so a
297/// Card reference never contains `[`, `]` or a second `/`, which is what
298/// lets `results[{card}][{metric}]` be split without escaping.
299#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
300#[serde(try_from = "String", into = "String")]
301pub struct CardRef(String);
302
303impl CardRef {
304    /// Parse `{ns}/{name}`, or `None` when it is not of that form.
305    pub fn parse(s: &str) -> Option<Self> {
306        evalhub_core::validate::is_valid_id(s).then(|| Self(s.to_string()))
307    }
308
309    /// The text form, `{ns}/{name}`.
310    pub fn as_str(&self) -> &str {
311        &self.0
312    }
313
314    /// The namespace part.
315    pub fn ns(&self) -> &str {
316        self.0.split_once('/').map_or("", |(ns, _)| ns)
317    }
318
319    /// The name part.
320    pub fn name(&self) -> &str {
321        self.0.split_once('/').map_or("", |(_, name)| name)
322    }
323}
324
325impl std::fmt::Display for CardRef {
326    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
327        f.write_str(&self.0)
328    }
329}
330
331impl std::str::FromStr for CardRef {
332    type Err = InvalidCardRef;
333    fn from_str(s: &str) -> Result<Self, Self::Err> {
334        Self::parse(s).ok_or_else(|| InvalidCardRef(s.to_string()))
335    }
336}
337
338impl TryFrom<String> for CardRef {
339    type Error = InvalidCardRef;
340    fn try_from(s: String) -> Result<Self, Self::Error> {
341        s.parse()
342    }
343}
344
345impl From<CardRef> for String {
346    fn from(card: CardRef) -> Self {
347        card.0
348    }
349}
350
351/// The text was not a Card reference of the form `{ns}/{name}`.
352#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
353#[error("not a Card reference of the form {{ns}}/{{name}}: {0:?}")]
354pub struct InvalidCardRef(pub String);
355
356/// The columns an `any` condition may compare, one set per table.
357#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
358#[serde(rename_all = "snake_case")]
359pub enum ArrayColumn {
360    /// `results.metric`
361    ResultMetric,
362    /// `results.aggregation`
363    ResultAggregation,
364    /// `results.value`
365    ResultValue,
366    /// `results.n`
367    ResultN,
368    /// `relations.type`
369    RelationType,
370    /// The relation's target, resolved or textual.
371    RelationTo,
372    /// `attachment_refs.path`
373    AttachmentPath,
374    /// `attachment_refs.sha256`
375    AttachmentSha256,
376}
377
378impl ArrayColumn {
379    /// The table this column belongs to, so a compiler can reject a
380    /// condition that strayed into another one.
381    pub fn table(self) -> ArrayTable {
382        match self {
383            Self::ResultMetric | Self::ResultAggregation | Self::ResultValue | Self::ResultN => {
384                ArrayTable::Results
385            }
386            Self::RelationType | Self::RelationTo => ArrayTable::Relations,
387            Self::AttachmentPath | Self::AttachmentSha256 => ArrayTable::Attachments,
388        }
389    }
390}
391
392/// The scalar types the query language distinguishes. They decide the cast
393/// helper for an `ext` path and the binding a literal gets.
394#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
395#[serde(rename_all = "snake_case")]
396pub enum ValueType {
397    /// IEEE 754 double, as every JSON number is.
398    Number,
399    /// UTF-8 text.
400    String,
401    /// `true` or `false`.
402    Boolean,
403}
404
405/// One sort key.
406#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
407pub struct Sort {
408    /// What to sort by.
409    pub key: SortKey,
410    /// Which way.
411    pub dir: Dir,
412}
413
414/// What a sort orders by.
415#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
416#[serde(rename_all = "snake_case")]
417pub enum SortKey {
418    /// A column of `versions`, or a registered `ext` path.
419    Column(Column),
420    /// The value of one metric, which is a lateral join on `results`.
421    Metric(String),
422    /// When the version was stored.
423    CreatedAt,
424    /// A column of the run projection. Only a [`RunQuery`] carries these;
425    /// a run query sorts on a registered `ext` path of the run with
426    /// [`SortKey::Column`] and [`Column::Ext`].
427    Run(RunColumn),
428}
429
430/// Sort direction.
431#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
432#[serde(rename_all = "snake_case")]
433pub enum Dir {
434    /// Smallest first.
435    Asc,
436    /// Largest first.
437    Desc,
438}
439
440/// Where the previous page stopped: the sort keys of its last row and that
441/// row's `version_id`, which breaks ties and makes the tuple unique.
442///
443/// The server signs and encodes this; the store only compares against it.
444#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
445pub struct Cursor {
446    /// One value per entry of [`Query::sort`], in the same order.
447    pub keys: Vec<Value>,
448    /// The last row's version id.
449    pub version_id: uuid::Uuid,
450}
451
452/// A checked, resolved read of one Eval's run projection, ready for a
453/// backend to compile.
454///
455/// Which Eval, and whether archived and deleted runs are included
456/// (`include=archived,deleted`, members only), are the server's to decide
457/// and pass beside this; the DSL knows neither. With no sort the backend
458/// orders by `run_id` ascending.
459#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
460pub struct RunQuery {
461    /// The Cards whose `run_results` are joined, in the order the request
462    /// named them. Every [`RunColumn::Result`] in the filter and the sort
463    /// names one of these; the projection carries all of them whether or
464    /// not a condition mentions them.
465    pub cards: Vec<CardRef>,
466    /// The predicate. `None` matches every run the server lets through.
467    pub filter: Option<Filter>,
468    /// Sort keys, most significant first. `run_id` is always the final,
469    /// implicit tie-breaker.
470    pub sort: Vec<Sort>,
471    /// Page size the caller asked for, already clamped by the server.
472    pub limit: u32,
473    /// Where the previous page stopped, decoded and verified by the server.
474    pub cursor: Option<RunCursor>,
475}
476
477/// Where the previous page of a run projection stopped: the sort keys of
478/// its last row and that row's `run_id`, which breaks ties and makes the
479/// tuple unique within the Eval.
480///
481/// [`Cursor`] cannot serve here, because its tie-breaker is a
482/// `version_id` and a run is not a version. The server treats this exactly
483/// as it treats a [`Cursor`]: serialises it as JSON, signs the bytes, and
484/// hands the client an opaque string; on the next request it verifies the
485/// signature and deserialises. The store only compares against it.
486#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
487pub struct RunCursor {
488    /// One value per entry of [`RunQuery::sort`], in the same order. A run
489    /// with no value for a key (a metric it lacks) carries `null`.
490    pub keys: Vec<Value>,
491    /// The last row's `run_id`.
492    pub run_id: String,
493}