1use std::collections::BTreeMap;
44
45use evalhub_schema::RecordKind;
46use evalhub_schema::error::{ErrorCode, ErrorEntry};
47use evalhub_schema::query::{self as envelope, QueryRequest};
48use serde_json::Value;
49
50use crate::grammar::{self, Filter, MatchTerm, ScalarOp};
51use crate::ir;
52
53const CARD_FACETS: [&str; 7] = [
56 "model",
57 "task",
58 "harness",
59 "generation",
60 "trial",
61 "grading",
62 "env",
63];
64
65pub const GENERATED_COLUMNS: &[(&[&str], &str)] = &[
73 (&["title"], "title"),
74 (&["producer", "name"], "producer_name"),
75 (&["model", "id"], "model_id"),
76 (&["model", "provider"], "model_provider"),
77 (&["model", "revision"], "model_revision"),
78 (&["task", "id"], "task_id"),
79 (&["task", "version"], "task_version"),
80 (&["task", "split"], "task_split"),
81 (&["harness", "name"], "harness_name"),
82 (&["harness", "version"], "harness_version"),
83 (&["generation", "temperature"], "gen_temperature"),
84 (&["generation", "top_p"], "gen_top_p"),
85 (&["generation", "max_tokens"], "gen_max_tokens"),
86 (&["trial", "k"], "trial_k"),
87 (&["env", "git", "commit"], "env_git_commit"),
88 (&["env", "git", "dirty"], "env_git_dirty"),
89];
90
91#[derive(Debug, Clone, Copy, PartialEq, Eq)]
93pub enum Indexed {
94 Full,
96 EqExistsOnly,
100 EqExistsUntyped,
109}
110
111#[derive(Debug, Clone, PartialEq)]
119pub enum PathInfo {
120 Scalar {
122 column: ir::Column,
124 ty: ir::ValueType,
126 indexed: Indexed,
128 },
129 Array {
131 table: ir::ArrayTable,
133 },
134}
135
136#[derive(Debug, Clone, PartialEq)]
138pub struct ExtSchema {
139 pub ns: String,
141 pub paths: Vec<(Vec<String>, ir::ValueType)>,
143 pub applied: bool,
146}
147
148#[derive(Debug, Default, Clone)]
151pub struct PathTable {
152 paths: BTreeMap<String, PathInfo>,
153}
154
155impl PathTable {
156 pub fn from_schema(kind: RecordKind) -> Self {
158 let mut paths = BTreeMap::new();
159 let schema = kind.schema().to_value();
160
161 let root_props = schema
163 .get("properties")
164 .and_then(Value::as_object)
165 .cloned()
166 .unwrap_or_default();
167 let defs = schema.get("$defs").cloned().unwrap_or(Value::Null);
168
169 for (name, prop) in &root_props {
170 match name.as_str() {
171 "ext" => {}
174 "results" => {
175 paths.insert(
176 name.clone(),
177 PathInfo::Array {
178 table: ir::ArrayTable::Results,
179 },
180 );
181 }
182 "relations" => {
183 paths.insert(
184 name.clone(),
185 PathInfo::Array {
186 table: ir::ArrayTable::Relations,
187 },
188 );
189 }
190 "attachments" => {
191 paths.insert(
192 name.clone(),
193 PathInfo::Array {
194 table: ir::ArrayTable::Attachments,
195 },
196 );
197 }
198 "runs" => {}
200 _ => walk(&mut paths, &defs, prop, std::slice::from_ref(name)),
201 }
202 }
203
204 for facet in facets(kind) {
206 paths.insert(
207 format!("fingerprint.{facet}"),
208 PathInfo::Scalar {
209 column: ir::Column::Fingerprint((*facet).to_string()),
210 ty: ir::ValueType::String,
211 indexed: Indexed::Full,
212 },
213 );
214 }
215
216 Self { paths }
217 }
218
219 pub fn with_ext(&self, entries: &[ExtSchema]) -> Self {
224 let mut paths = self.paths.clone();
225 for entry in entries.iter().filter(|e| e.applied) {
226 for (segments, ty) in &entry.paths {
227 let path = format!("ext.{}.{}", entry.ns, segments.join("."));
228 let mut json_path = vec!["ext".to_string(), entry.ns.clone()];
229 json_path.extend(segments.iter().cloned());
230 paths.insert(
231 path,
232 PathInfo::Scalar {
233 column: ir::Column::Ext {
234 path: json_path,
235 ty: *ty,
236 },
237 ty: *ty,
238 indexed: Indexed::Full,
239 },
240 );
241 }
242 }
243 Self { paths }
244 }
245
246 pub fn lookup(&self, path: &str) -> Option<PathInfo> {
252 if let Some(info) = self.paths.get(path) {
253 return Some(info.clone());
254 }
255 if let Some(rest) = path.strip_prefix("ext.") {
256 let (ns, key) = rest.split_once('.')?;
259 if ns.is_empty() || key.is_empty() {
260 return None;
261 }
262 let mut json_path = vec!["ext".to_string(), ns.to_string()];
263 json_path.extend(key.split('.').map(str::to_string));
264 return Some(PathInfo::Scalar {
265 column: ir::Column::Body(json_path),
266 ty: ir::ValueType::String,
269 indexed: Indexed::EqExistsUntyped,
270 });
271 }
272 None
273 }
274
275 pub fn paths(&self) -> impl Iterator<Item = (&str, &PathInfo)> {
278 self.paths.iter().map(|(k, v)| (k.as_str(), v))
279 }
280}
281
282fn facets(kind: RecordKind) -> &'static [&'static str] {
284 match kind {
285 RecordKind::Card => &CARD_FACETS,
286 RecordKind::Eval => &["model", "task", "harness", "generation", "trial", "env"],
287 }
288}
289
290fn walk(paths: &mut BTreeMap<String, PathInfo>, defs: &Value, prop: &Value, segments: &[String]) {
292 let resolved = resolve(defs, prop);
293 let Some(resolved) = resolved else { return };
294
295 if let Some(properties) = resolved.get("properties").and_then(Value::as_object) {
296 for (name, child) in properties {
297 if name == "ext" {
300 continue;
301 }
302 let mut next = segments.to_vec();
303 next.push(name.clone());
304 walk(paths, defs, child, &next);
305 }
306 return;
307 }
308
309 let Some(ty) = scalar_type(&resolved) else {
310 return;
314 };
315 let path = segments.join(".");
316 let column = GENERATED_COLUMNS
317 .iter()
318 .find(|(json_path, _)| json_path == &segments)
319 .map(|(_, name)| ir::Column::Generated((*name).to_string()));
320 let (column, indexed) = match column {
321 Some(column) => (column, Indexed::Full),
322 None => (ir::Column::Body(segments.to_vec()), Indexed::EqExistsOnly),
323 };
324 paths.insert(
325 path,
326 PathInfo::Scalar {
327 column,
328 ty,
329 indexed,
330 },
331 );
332}
333
334fn resolve(defs: &Value, prop: &Value) -> Option<Value> {
337 if let Some(reference) = prop.get("$ref").and_then(Value::as_str) {
338 let name = reference.rsplit('/').next()?;
339 return defs.get(name).cloned();
340 }
341 if let Some(any_of) = prop.get("anyOf").and_then(Value::as_array) {
342 for branch in any_of {
345 if branch.get("type").and_then(Value::as_str) == Some("null") {
346 continue;
347 }
348 if let Some(resolved) = resolve(defs, branch) {
349 return Some(resolved);
350 }
351 }
352 return None;
353 }
354 Some(prop.clone())
355}
356
357fn scalar_type(schema: &Value) -> Option<ir::ValueType> {
359 let ty = schema.get("type")?;
360 let names: Vec<&str> = match ty {
361 Value::String(s) => vec![s.as_str()],
362 Value::Array(items) => items.iter().filter_map(Value::as_str).collect(),
363 _ => return None,
364 };
365 for name in names {
366 match name {
367 "string" => return Some(ir::ValueType::String),
368 "number" | "integer" => return Some(ir::ValueType::Number),
369 "boolean" => return Some(ir::ValueType::Boolean),
370 _ => continue,
371 }
372 }
373 None
374}
375
376fn array_column(table: ir::ArrayTable, key: &str) -> Option<(ir::ArrayColumn, ir::ValueType)> {
378 use ir::{ArrayColumn as C, ArrayTable as T, ValueType as V};
379 match (table, key) {
380 (T::Results, "metric") => Some((C::ResultMetric, V::String)),
381 (T::Results, "aggregation") => Some((C::ResultAggregation, V::String)),
382 (T::Results, "value") => Some((C::ResultValue, V::Number)),
383 (T::Results, "n") => Some((C::ResultN, V::Number)),
384 (T::Relations, "type") => Some((C::RelationType, V::String)),
385 (T::Relations, "to") => Some((C::RelationTo, V::String)),
386 (T::Attachments, "path") => Some((C::AttachmentPath, V::String)),
387 (T::Attachments, "sha256") => Some((C::AttachmentSha256, V::String)),
388 _ => None,
389 }
390}
391
392fn array_keys(table: ir::ArrayTable) -> &'static [&'static str] {
394 match table {
395 ir::ArrayTable::Results => &["metric", "aggregation", "value", "n"],
396 ir::ArrayTable::Relations => &["type", "to"],
397 ir::ArrayTable::Attachments => &["path", "sha256"],
398 }
399}
400
401fn entry(path: &str, code: ErrorCode, hint: impl Into<String>) -> ErrorEntry {
402 ErrorEntry {
403 path: path.to_string(),
404 code,
405 hint: Some(hint.into()),
406 }
407}
408
409fn op_suits(op: ScalarOp, ty: ir::ValueType) -> bool {
411 use ir::ValueType as V;
412 match op {
413 ScalarOp::Eq | ScalarOp::Ne | ScalarOp::In => true,
414 ScalarOp::Gt | ScalarOp::Gte | ScalarOp::Lt | ScalarOp::Lte => {
415 matches!(ty, V::Number | V::String)
416 }
417 ScalarOp::Prefix | ScalarOp::Contains => matches!(ty, V::String),
418 }
419}
420
421fn index_serves(indexed: Indexed, op: ScalarOp) -> bool {
423 match indexed {
424 Indexed::Full => true,
425 Indexed::EqExistsOnly | Indexed::EqExistsUntyped => matches!(op, ScalarOp::Eq),
426 }
427}
428
429fn literal_matches(value: &Value, ty: ir::ValueType) -> bool {
431 match ty {
432 ir::ValueType::Number => value.is_number(),
433 ir::ValueType::String => value.is_string(),
434 ir::ValueType::Boolean => value.is_boolean(),
435 }
436}
437
438fn type_name(ty: ir::ValueType) -> &'static str {
439 match ty {
440 ir::ValueType::Number => "a number",
441 ir::ValueType::String => "a string",
442 ir::ValueType::Boolean => "a boolean",
443 }
444}
445
446fn check_cmp(
449 column: ir::Column,
450 ty: ir::ValueType,
451 indexed: Indexed,
452 op: ScalarOp,
453 value: &Value,
454 pointer: &str,
455 errors: &mut Vec<ErrorEntry>,
456) -> Option<ir::Cmp> {
457 let mut ok = true;
458 let typed = indexed != Indexed::EqExistsUntyped;
461 if typed && !op_suits(op, ty) {
462 errors.push(entry(
463 &format!("{pointer}/op"),
464 ErrorCode::TypeMismatch,
465 format!("`{op}` does not apply to {}", type_name(ty)),
466 ));
467 ok = false;
468 } else if !index_serves(indexed, op) {
469 errors.push(entry(
470 &format!("{pointer}/op"),
471 ErrorCode::NotIndexed,
472 format!(
473 "`{op}` needs an index this path does not have; it answers `eq` and `exists` \
474 from the index over the whole record. Register an `ext_schema` for it in the \
475 registry (`PUT /registry/ext_schemas/{{ns}}/{{id}}@{{version}}`) to make it \
476 sortable and rangeable."
477 ),
478 ));
479 ok = false;
480 }
481
482 if op == ScalarOp::In {
484 match value.as_array() {
485 None => {
486 errors.push(entry(
487 &format!("{pointer}/value"),
488 ErrorCode::TypeMismatch,
489 "`in` takes an array",
490 ));
491 ok = false;
492 }
493 Some(items) => {
494 if let Some(bad) = items.iter().position(|v| typed && !literal_matches(v, ty)) {
495 errors.push(entry(
496 &format!("{pointer}/value/{bad}"),
497 ErrorCode::TypeMismatch,
498 format!("this path holds {}", type_name(ty)),
499 ));
500 ok = false;
501 }
502 }
503 }
504 } else if typed && !literal_matches(value, ty) {
505 errors.push(entry(
506 &format!("{pointer}/value"),
507 ErrorCode::TypeMismatch,
508 format!("this path holds {}", type_name(ty)),
509 ));
510 ok = false;
511 }
512
513 ok.then(|| ir::Cmp {
514 column,
515 op: to_ir_op(op),
516 value: value.clone(),
517 })
518}
519
520fn to_ir_op(op: ScalarOp) -> ir::Op {
521 match op {
522 ScalarOp::Eq => ir::Op::Eq,
523 ScalarOp::Ne => ir::Op::Ne,
524 ScalarOp::Gt => ir::Op::Gt,
525 ScalarOp::Gte => ir::Op::Gte,
526 ScalarOp::Lt => ir::Op::Lt,
527 ScalarOp::Lte => ir::Op::Lte,
528 ScalarOp::In => ir::Op::In,
529 ScalarOp::Prefix => ir::Op::Prefix,
530 ScalarOp::Contains => ir::Op::Contains,
531 }
532}
533
534pub fn check(ast: &Filter, table: &PathTable) -> Result<ir::Filter, Vec<ErrorEntry>> {
540 let mut errors = Vec::new();
541 let filter = check_at(ast, table, "/where", &mut errors);
542 match (filter, errors.is_empty()) {
543 (Some(filter), true) => Ok(filter),
544 _ => Err(errors),
545 }
546}
547
548fn check_at(
549 ast: &Filter,
550 table: &PathTable,
551 pointer: &str,
552 errors: &mut Vec<ErrorEntry>,
553) -> Option<ir::Filter> {
554 match ast {
555 Filter::And { children, .. } | Filter::Or { children, .. } => {
556 let key = grammar::wire_key(ast);
557 let checked: Vec<ir::Filter> = children
558 .iter()
559 .enumerate()
560 .filter_map(|(i, child)| {
561 check_at(child, table, &format!("{pointer}/{key}/{i}"), errors)
562 })
563 .collect();
564 if checked.len() != children.len() {
565 return None;
566 }
567 Some(match ast {
568 Filter::And { .. } => ir::Filter::And(checked),
569 _ => ir::Filter::Or(checked),
570 })
571 }
572 Filter::Not { child, .. } => {
573 let inner = check_at(child, table, &format!("{pointer}/not"), errors)?;
574 Some(ir::Filter::Not(Box::new(inner)))
575 }
576 Filter::Compare {
577 path, op, value, ..
578 } => {
579 let info = lookup_or_report(table, path, pointer, errors)?;
580 match info {
581 PathInfo::Scalar {
582 column,
583 ty,
584 indexed,
585 } => {
586 check_cmp(column, ty, indexed, *op, value, pointer, errors).map(ir::Filter::Cmp)
587 }
588 PathInfo::Array { .. } => {
589 errors.push(entry(
590 &format!("{pointer}/op"),
591 ErrorCode::TypeMismatch,
592 format!("`{path}` is an array; search it with `any` and a `match`"),
593 ));
594 None
595 }
596 }
597 }
598 Filter::Exists { path, .. } => {
599 let info = lookup_or_report(table, path, pointer, errors)?;
600 match info {
601 PathInfo::Scalar { column, .. } => Some(ir::Filter::Exists { column }),
602 PathInfo::Array { .. } => {
603 errors.push(entry(
604 &format!("{pointer}/path"),
605 ErrorCode::TypeMismatch,
606 format!("`{path}` is an array; search it with `any` and a `match`"),
607 ));
608 None
609 }
610 }
611 }
612 Filter::Any { path, terms, .. } => {
613 let info = lookup_or_report(table, path, pointer, errors)?;
614 let PathInfo::Array { table: side } = info else {
615 errors.push(entry(
616 &format!("{pointer}/op"),
617 ErrorCode::TypeMismatch,
618 format!("`{path}` is not an array; `any` ranges over one"),
619 ));
620 return None;
621 };
622 check_terms(side, terms, pointer, errors).map(|conditions| ir::Filter::Any {
623 table: side,
624 conditions,
625 })
626 }
627 }
628}
629
630fn check_terms(
631 side: ir::ArrayTable,
632 terms: &BTreeMap<String, MatchTerm>,
633 pointer: &str,
634 errors: &mut Vec<ErrorEntry>,
635) -> Option<Vec<ir::Cmp>> {
636 if terms.is_empty() {
637 errors.push(entry(
638 &format!("{pointer}/match"),
639 ErrorCode::Schema,
640 format!(
641 "`match` needs at least one of {}",
642 array_keys(side)
643 .iter()
644 .map(|k| format!("`{k}`"))
645 .collect::<Vec<_>>()
646 .join(", ")
647 ),
648 ));
649 return None;
650 }
651 let mut conditions = Vec::new();
652 let mut ok = true;
653 for (key, term) in terms {
654 let term_pointer = format!("{pointer}/match/{key}");
655 let Some((column, ty)) = array_column(side, key) else {
656 errors.push(entry(
657 &term_pointer,
658 ErrorCode::UnknownPath,
659 format!(
660 "unknown key `{key}`; this array has {}",
661 array_keys(side)
662 .iter()
663 .map(|k| format!("`{k}`"))
664 .collect::<Vec<_>>()
665 .join(", ")
666 ),
667 ));
668 ok = false;
669 continue;
670 };
671 match check_cmp(
672 ir::Column::Array(column),
673 ty,
674 Indexed::Full,
675 term.op,
676 &term.value,
677 &term_pointer,
678 errors,
679 ) {
680 Some(cmp) => conditions.push(cmp),
681 None => ok = false,
682 }
683 }
684 ok.then_some(conditions)
685}
686
687fn lookup_or_report(
688 table: &PathTable,
689 path: &str,
690 pointer: &str,
691 errors: &mut Vec<ErrorEntry>,
692) -> Option<PathInfo> {
693 match table.lookup(path) {
694 Some(info) => Some(info),
695 None => {
696 errors.push(entry(
697 &format!("{pointer}/path"),
698 ErrorCode::UnknownPath,
699 format!("`{path}` is not a queryable path of this record"),
700 ));
701 None
702 }
703 }
704}
705
706pub fn compile(
712 request: &QueryRequest,
713 kind: RecordKind,
714 table: &PathTable,
715 limit: u32,
716 cursor: Option<ir::Cursor>,
717) -> Result<ir::Query, Vec<ErrorEntry>> {
718 let mut errors = Vec::new();
719
720 let filter = match &request.where_ {
721 None => None,
722 Some(value) => match grammar::parse(value) {
723 Ok(ast) => match check(&ast, table) {
724 Ok(filter) => Some(filter),
725 Err(mut e) => {
726 errors.append(&mut e);
727 None
728 }
729 },
730 Err(mut e) => {
731 errors.append(&mut e);
732 None
733 }
734 },
735 };
736
737 let mut sort = Vec::new();
738 for (i, key) in request.sort.iter().enumerate() {
739 let pointer = format!("/sort/{i}");
740 if let Some(checked) = check_sort(&key.path, table, &pointer, &mut errors) {
741 sort.push(ir::Sort {
742 key: checked,
743 dir: match key.dir {
744 envelope::Dir::Asc => ir::Dir::Asc,
745 envelope::Dir::Desc => ir::Dir::Desc,
746 },
747 });
748 }
749 }
750
751 if let Some(cursor) = &cursor
752 && cursor.keys.len() != sort.len()
753 {
754 errors.push(entry(
755 "/cursor",
756 ErrorCode::Schema,
757 "the cursor does not match this sort; start the page sequence again",
758 ));
759 }
760
761 if !errors.is_empty() {
762 return Err(errors);
763 }
764
765 Ok(ir::Query {
766 record_type: match kind {
767 RecordKind::Card => ir::RecordType::Card,
768 RecordKind::Eval => ir::RecordType::Eval,
769 },
770 filter,
771 sort,
772 limit,
773 cursor,
774 expand: request
775 .expand
776 .iter()
777 .map(|e| match e {
778 envelope::Expand::Relations => ir::Expand::Relations,
779 envelope::Expand::Badges => ir::Expand::Badges,
780 envelope::Expand::Fingerprints => ir::Expand::Fingerprints,
781 envelope::Expand::Changed => ir::Expand::Changed,
782 })
783 .collect(),
784 version: match request.version {
785 None | Some(envelope::VersionSelector::Latest) => ir::VersionSelector::Latest,
786 Some(envelope::VersionSelector::All) => ir::VersionSelector::All,
787 },
788 })
789}
790
791fn check_sort(
794 path: &str,
795 table: &PathTable,
796 pointer: &str,
797 errors: &mut Vec<ErrorEntry>,
798) -> Option<ir::SortKey> {
799 if path == "created_at" {
800 return Some(ir::SortKey::CreatedAt);
801 }
802 if let Some(rest) = path.strip_prefix("results[")
804 && let Some((metric, tail)) = rest.split_once(']')
805 {
806 if tail != ".value" {
807 errors.push(entry(
808 &format!("{pointer}/path"),
809 ErrorCode::UnknownPath,
810 "a metric sorts by its `value`: `results[{metric}].value`",
811 ));
812 return None;
813 }
814 if !metric.contains('/') {
815 errors.push(entry(
816 &format!("{pointer}/path"),
817 ErrorCode::UnknownPath,
818 format!("`{metric}` is not a metric id of the form `{{ns}}/{{name}}`"),
819 ));
820 return None;
821 }
822 return Some(ir::SortKey::Metric(metric.to_string()));
823 }
824
825 match table.lookup(path) {
826 Some(PathInfo::Scalar {
827 column,
828 indexed: Indexed::Full,
829 ..
830 }) => Some(ir::SortKey::Column(column)),
831 Some(PathInfo::Scalar { .. }) => {
832 errors.push(entry(
833 &format!("{pointer}/path"),
834 ErrorCode::NotIndexed,
835 format!(
836 "`{path}` has no ordered index, so it cannot be sorted on. Register an \
837 `ext_schema` for it in the registry to make it sortable."
838 ),
839 ));
840 None
841 }
842 Some(PathInfo::Array { .. }) => {
843 errors.push(entry(
844 &format!("{pointer}/path"),
845 ErrorCode::TypeMismatch,
846 format!("`{path}` is an array; sort by `results[{{metric}}].value` instead"),
847 ));
848 None
849 }
850 None => {
851 errors.push(entry(
852 &format!("{pointer}/path"),
853 ErrorCode::UnknownPath,
854 format!("`{path}` is not a queryable path of this record"),
855 ));
856 None
857 }
858 }
859}