1use std::collections::HashSet;
57use std::sync::OnceLock;
58
59use jsonschema::{Draft, Validator};
60use serde_json::Value;
61
62use evalhub_schema::RecordKind;
63use evalhub_schema::error::{ErrorCode, ErrorEntry};
64
65const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
68
69#[derive(Debug, Clone, PartialEq, Eq)]
71pub enum RelationTarget {
72 Version {
74 ns: String,
76 name: String,
78 seq: u32,
80 },
81 External(String),
83 Hf(String),
85}
86
87pub fn is_valid_id(s: &str) -> bool {
91 let Some((ns, name)) = s.split_once('/') else {
92 return false;
93 };
94 is_slug(ns) && is_slug(name)
95}
96
97fn is_slug(part: &str) -> bool {
98 let mut chars = part.chars();
99 match chars.next() {
100 Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
101 _ => return false,
102 }
103 chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
104}
105
106pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
109 if let Some(rest) = s.strip_prefix("external:") {
110 return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
111 }
112 if let Some(rest) = s.strip_prefix("hf:") {
113 return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
114 }
115 let (id, seq) = s.rsplit_once('@')?;
116 let (ns, name) = id.split_once('/')?;
117 if !is_slug(ns) || !is_slug(name) {
118 return None;
119 }
120 if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
122 return None;
123 }
124 let seq: u32 = seq.parse().ok()?;
125 if seq == 0 {
126 return None;
127 }
128 Some(RelationTarget::Version {
129 ns: ns.to_string(),
130 name: name.to_string(),
131 seq,
132 })
133}
134
135pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
137 let mut errors = Vec::new();
138 structural(kind, value, &mut errors);
139 numbers("", value, &mut errors);
140 semantic(kind, value, &mut errors);
141 errors.sort_by(|a, b| {
143 a.path
144 .cmp(&b.path)
145 .then_with(|| (a.code as u8).cmp(&(b.code as u8)))
146 });
147 errors
148}
149
150fn validator(kind: RecordKind) -> &'static Validator {
151 static CARD: OnceLock<Validator> = OnceLock::new();
152 static EVAL: OnceLock<Validator> = OnceLock::new();
153 let cell = match kind {
154 RecordKind::Card => &CARD,
155 RecordKind::Eval => &EVAL,
156 };
157 cell.get_or_init(|| {
158 let schema = kind.schema().to_value();
159 #[allow(clippy::expect_used)]
163 jsonschema::options()
164 .with_draft(Draft::Draft202012)
165 .offline()
166 .build(&schema)
167 .expect("the generated record schema compiles")
168 })
169}
170
171fn structural(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
172 for e in validator(kind).iter_errors(value) {
173 out.push(ErrorEntry {
174 path: e.instance_path().as_str().to_string(),
175 code: ErrorCode::Schema,
176 hint: Some(e.to_string()),
177 });
178 }
179}
180
181fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
182 match value {
183 Value::Number(n) => {
184 let too_large = if let Some(i) = n.as_i64() {
185 i.unsigned_abs() > MAX_SAFE_INTEGER as u64
186 } else if let Some(u) = n.as_u64() {
187 u > MAX_SAFE_INTEGER as u64
188 } else if let Some(f) = n.as_f64() {
189 f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
190 } else {
191 false
192 };
193 if too_large {
194 out.push(ErrorEntry {
195 path: path.to_string(),
196 code: ErrorCode::NumberTooLarge,
197 hint: Some(format!(
198 "{n} is an integer outside ±2^53; send it as a string"
199 )),
200 });
201 }
202 }
203 Value::Array(items) => {
204 for (i, item) in items.iter().enumerate() {
205 numbers(&format!("{path}/{i}"), item, out);
206 }
207 }
208 Value::Object(map) => {
209 for (k, v) in map {
210 numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
211 }
212 }
213 _ => {}
214 }
215}
216
217fn escape_pointer(key: &str) -> String {
219 key.replace('~', "~0").replace('/', "~1")
220}
221
222fn semantic(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
223 let Some(obj) = value.as_object() else {
224 return;
225 };
226 let push = |out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String| {
227 out.push(ErrorEntry {
228 path,
229 code,
230 hint: Some(hint),
231 })
232 };
233
234 let mut paths: HashSet<&str> = HashSet::new();
236 if let Some(attachments) = obj.get("attachments").and_then(Value::as_array) {
237 for (i, a) in attachments.iter().enumerate() {
238 let Some(p) = a.get("path").and_then(Value::as_str) else {
239 continue;
240 };
241 let pointer = format!("/attachments/{i}/path");
242 let invalid = if p.is_empty() {
243 Some("empty")
244 } else if p.starts_with('/') {
245 Some("absolute")
246 } else if p.split('/').any(|seg| seg == "..") {
247 Some("contains `..`")
248 } else if !paths.insert(p) {
249 Some("duplicate")
250 } else {
251 None
252 };
253 if let Some(why) = invalid {
254 push(
255 out,
256 pointer,
257 ErrorCode::AttachmentPathInvalid,
258 format!("attachment path {p:?} is {why}"),
259 );
260 }
261 }
262 }
263 let ref_unknown = |out: &mut Vec<ErrorEntry>, pointer: String, p: &str| {
264 if !paths.contains(p) {
265 push(
266 out,
267 pointer,
268 ErrorCode::AttachmentRefUnknown,
269 format!("{p:?} names no attachments[].path"),
270 );
271 }
272 };
273
274 if kind == RecordKind::Card {
275 let results = obj.get("results").and_then(Value::as_array);
276 if let Some(results) = results {
277 for (i, r) in results.iter().enumerate() {
278 if let Some(m) = r.get("metric").and_then(Value::as_str)
279 && !is_valid_id(m)
280 {
281 push(
282 out,
283 format!("/results/{i}/metric"),
284 ErrorCode::MetricIdInvalid,
285 format!("{m:?} is not of the form {{ns}}/{{name}}"),
286 );
287 }
288 if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
289 ref_unknown(out, format!("/results/{i}/samples_ref"), s);
290 }
291 }
292 let counts = obj.get("counts").filter(|c| !c.is_null());
293 match counts {
294 None if !results.is_empty() => push(
295 out,
296 "/counts".to_string(),
297 ErrorCode::CountsMissing,
298 "results are present, so counts is required".to_string(),
299 ),
300 Some(c) => {
301 let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
302 let sum = n("completed") + n("failed") + n("skipped") + n("errored");
303 if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
304 && attempted < sum
305 {
306 push(
307 out,
308 "/counts".to_string(),
309 ErrorCode::CountsInconsistent,
310 format!(
311 "attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
312 ),
313 );
314 }
315 }
316 None => {}
317 }
318 }
319 }
320
321 if kind == RecordKind::Eval
322 && let Some(runs) = obj.get("runs").and_then(Value::as_array)
323 {
324 for (i, run) in runs.iter().enumerate() {
325 if let Some(c) = run.get("calls").and_then(Value::as_str) {
326 ref_unknown(out, format!("/runs/{i}/calls"), c);
327 }
328 if let Some(artifacts) = run.get("artifacts").and_then(Value::as_array) {
329 for (j, a) in artifacts.iter().enumerate() {
330 if let Some(a) = a.as_str() {
331 ref_unknown(out, format!("/runs/{i}/artifacts/{j}"), a);
332 }
333 }
334 }
335 }
336 }
337
338 if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
339 for (i, rel) in relations.iter().enumerate() {
340 if let Some(t) = rel.get("type").and_then(Value::as_str)
341 && !is_valid_id(t)
342 {
343 push(
344 out,
345 format!("/relations/{i}/type"),
346 ErrorCode::Schema,
347 format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
348 );
349 }
350 if let Some(to) = rel.get("to").and_then(Value::as_str)
351 && parse_relation_target(to).is_none()
352 {
353 push(
354 out,
355 format!("/relations/{i}/to"),
356 ErrorCode::Schema,
357 format!(
358 "{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
359 ),
360 );
361 }
362 }
363 }
364}
365
366#[cfg(test)]
367mod tests {
368 #![allow(clippy::unwrap_used)]
369
370 use super::*;
371
372 #[test]
373 fn ids() {
374 assert!(is_valid_id("core/pass_rate"));
375 assert!(is_valid_id("alice/qwen-loop.v2"));
376 assert!(!is_valid_id("pass_rate"));
377 assert!(!is_valid_id("Core/pass_rate"));
378 assert!(!is_valid_id("core/"));
379 assert!(!is_valid_id("/x"));
380 assert!(!is_valid_id("a/b/c"));
381 assert!(!is_valid_id("-a/b"));
382 }
383
384 #[test]
385 fn relation_targets() {
386 assert_eq!(
387 parse_relation_target("alice/single2-k4@3"),
388 Some(RelationTarget::Version {
389 ns: "alice".into(),
390 name: "single2-k4".into(),
391 seq: 3
392 })
393 );
394 assert_eq!(
395 parse_relation_target("external:https://x/y"),
396 Some(RelationTarget::External("https://x/y".into()))
397 );
398 assert_eq!(
399 parse_relation_target("hf:org/repo@sha"),
400 Some(RelationTarget::Hf("org/repo@sha".into()))
401 );
402 for bad in [
403 "alice/x",
404 "alice/x@0",
405 "alice/x@+1",
406 "alice/x@label",
407 "alice@3",
408 "external:",
409 "hf:",
410 "",
411 ] {
412 assert_eq!(parse_relation_target(bad), None, "{bad}");
413 }
414 }
415
416 #[test]
417 fn validators_build_offline() {
418 let _ = validator(RecordKind::Card);
419 let _ = validator(RecordKind::Eval);
420 }
421
422 #[test]
423 fn pointer_escaping() {
424 let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
425 let mut out = Vec::new();
426 numbers("", &v, &mut out);
427 assert_eq!(out.len(), 1);
428 assert_eq!(out[0].path, "/ext/a~1b/n");
429 assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
430 }
431}