1use std::collections::{BTreeMap, BTreeSet};
10use std::fmt::Write as _;
11
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14use sha2::{Digest, Sha256};
15
16use crate::error::{EvalError, Result};
17use crate::evidence::RunStatus;
18use crate::experiment::{CaseRunReport, ExperimentReport};
19
20pub const FAILURE_CLUSTER_REPORT_FORMAT_VERSION: u64 = 1;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
25#[serde(rename_all = "snake_case")]
26pub enum FailureTermination {
27 Done,
29 Interrupted,
31 Failed,
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
37#[serde(rename_all = "snake_case")]
38pub enum ExecutionFailureCategory {
39 Authentication,
41 Authorization,
43 Timeout,
45 RateLimit,
47 Transport,
49 Unavailable,
51 InvalidData,
53 Cancelled,
55 Unknown,
57}
58
59#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
61#[serde(tag = "cause", rename_all = "snake_case")]
62pub enum FailureCause {
63 Evaluation {
65 fingerprint: String,
67 },
68 Interrupted,
70 Execution {
72 category: ExecutionFailureCategory,
74 fingerprint: String,
76 },
77}
78
79#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
81pub struct AssertionFailureKey {
82 pub assertion: String,
84 pub expected_fingerprint: String,
86}
87
88#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
90pub struct FailureSignature {
91 pub termination: FailureTermination,
93 pub cause: FailureCause,
95 pub failed_assertions: Vec<AssertionFailureKey>,
97 pub judge_fingerprint: Option<String>,
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
103#[serde(tag = "evidence", rename_all = "snake_case")]
104pub enum FailureEvidenceRef {
105 Execution {
107 category: ExecutionFailureCategory,
109 fingerprint: String,
111 },
112 Interrupted,
114 Assertion {
116 assertion: String,
118 expected_fingerprint: String,
120 },
121 Judge {
123 rationale_fingerprint: String,
125 },
126}
127
128#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
130pub struct FailureOccurrence {
131 pub case_id: String,
133 pub repetition: usize,
135 pub tags: Vec<String>,
137 pub evidence: Vec<FailureEvidenceRef>,
139}
140
141#[derive(Debug, Clone, PartialEq, Serialize)]
143pub struct FailureCluster {
144 pub id: String,
146 pub signature: FailureSignature,
148 pub occurrences: usize,
150 pub share_of_failures: f64,
152 pub members: Vec<FailureOccurrence>,
154}
155
156#[derive(Debug, Clone, PartialEq, Serialize)]
158pub struct FailureClusterReport {
159 pub format_version: u64,
161 pub experiment: String,
163 pub dataset_name: String,
165 pub dataset_version: String,
167 pub total_runs: usize,
169 pub total_failures: usize,
171 pub clusters: Vec<FailureCluster>,
173}
174
175impl FailureClusterReport {
176 pub fn to_json(&self) -> Result<String> {
178 self.validate()?;
179 Ok(serde_json::to_string_pretty(self)?)
180 }
181
182 pub fn from_json(text: &str, source: &ExperimentReport) -> Result<Self> {
188 let header: FormatHeader = serde_json::from_str(text)?;
189 if header.format_version != FAILURE_CLUSTER_REPORT_FORMAT_VERSION {
190 return clustering_error(format!(
191 "unsupported failure cluster format version: found {}, this build supports {}",
192 header.format_version, FAILURE_CLUSTER_REPORT_FORMAT_VERSION
193 ));
194 }
195 let supplied = FailureClusterReportWire::deserialize_report(text)?;
196 supplied.validate()?;
197 let expected = cluster_failures(source)?;
198 if !reports_match(&supplied, &expected) {
199 return clustering_error("failure cluster report does not match its source evidence");
200 }
201 Ok(expected)
202 }
203
204 fn validate(&self) -> Result<()> {
205 if self.format_version != FAILURE_CLUSTER_REPORT_FORMAT_VERSION {
206 return clustering_error(format!(
207 "unsupported failure cluster format version: found {}, this build supports {}",
208 self.format_version, FAILURE_CLUSTER_REPORT_FORMAT_VERSION
209 ));
210 }
211 require_non_empty("experiment", &self.experiment)?;
212 require_non_empty("dataset name", &self.dataset_name)?;
213 require_non_empty("dataset version", &self.dataset_version)?;
214 if self.total_failures > self.total_runs {
215 return clustering_error("total failures cannot exceed total runs");
216 }
217 if (self.total_failures == 0) != self.clusters.is_empty() {
218 return clustering_error("clusters must be empty exactly when there are no failures");
219 }
220
221 let mut signatures = BTreeSet::new();
222 let mut ids = BTreeSet::new();
223 let mut occurrences = BTreeSet::new();
224 let mut counted_failures = 0_usize;
225 for cluster in &self.clusters {
226 validate_signature(&cluster.signature)?;
227 let expected_id = signature_id(&cluster.signature);
228 if cluster.id != expected_id {
229 return clustering_error(format!(
230 "cluster id `{}` does not match its signature",
231 cluster.id
232 ));
233 }
234 if !ids.insert(cluster.id.as_str()) || !signatures.insert(&cluster.signature) {
235 return clustering_error("cluster ids and signatures must be unique");
236 }
237 if cluster.occurrences == 0 || cluster.occurrences != cluster.members.len() {
238 return clustering_error(format!(
239 "cluster `{}` occurrence count does not match its members",
240 cluster.id
241 ));
242 }
243 counted_failures = counted_failures
244 .checked_add(cluster.occurrences)
245 .ok_or_else(|| EvalError::Clustering("failure count overflows usize".to_owned()))?;
246 let expected_share = cluster.occurrences as f64 / self.total_failures as f64;
247 if !same_float(cluster.share_of_failures, expected_share) {
248 return clustering_error(format!(
249 "cluster `{}` share does not match its occurrence count",
250 cluster.id
251 ));
252 }
253 if !cluster
254 .members
255 .windows(2)
256 .all(|pair| member_key(&pair[0]) < member_key(&pair[1]))
257 {
258 return clustering_error(format!(
259 "cluster `{}` members are not in canonical order",
260 cluster.id
261 ));
262 }
263 for member in &cluster.members {
264 require_non_empty("failure case id", &member.case_id)?;
265 if member.evidence.is_empty()
266 || !is_sorted_unique(&member.evidence)
267 || !is_sorted_unique(&member.tags)
268 {
269 return clustering_error(format!(
270 "cluster `{}` contains non-canonical member evidence",
271 cluster.id
272 ));
273 }
274 let member_signature = signature_from_evidence(&member.evidence)?;
275 if member_signature != cluster.signature {
276 return clustering_error(format!(
277 "cluster `{}` member evidence does not match its signature",
278 cluster.id
279 ));
280 }
281 if !occurrences.insert((member.case_id.as_str(), member.repetition)) {
282 return clustering_error(format!(
283 "failure occurrence `{}#{}` appears more than once",
284 member.case_id, member.repetition
285 ));
286 }
287 }
288 }
289 if counted_failures != self.total_failures {
290 return clustering_error("cluster occurrence counts do not equal total failures");
291 }
292 if !self
293 .clusters
294 .windows(2)
295 .all(|pair| cluster_sort_key(&pair[0]) < cluster_sort_key(&pair[1]))
296 {
297 return clustering_error("clusters are not in canonical rank order");
298 }
299 Ok(())
300 }
301}
302
303pub fn cluster_failures(source: &ExperimentReport) -> Result<FailureClusterReport> {
305 crate::gate::validate_report(source)
306 .map_err(|error| EvalError::Clustering(format!("source report is invalid: {error}")))?;
307
308 let mut grouped: BTreeMap<FailureSignature, Vec<FailureOccurrence>> = BTreeMap::new();
309 for case in &source.cases {
310 for run in &case.runs {
311 if run.passed {
312 continue;
313 }
314 let mut evidence = evidence_refs(run);
315 evidence.sort();
316 evidence.dedup();
317 let signature = signature_from_evidence(&evidence)?;
318 let mut tags = case.tags.clone();
319 tags.sort();
320 tags.dedup();
321 grouped
322 .entry(signature)
323 .or_default()
324 .push(FailureOccurrence {
325 case_id: case.case_id.clone(),
326 repetition: run.repetition,
327 tags,
328 evidence,
329 });
330 }
331 }
332
333 let total_failures = grouped.values().map(Vec::len).sum();
334 let mut clusters: Vec<_> = grouped
335 .into_iter()
336 .map(|(signature, mut members)| {
337 members.sort_by(|left, right| member_key(left).cmp(&member_key(right)));
338 let occurrences = members.len();
339 FailureCluster {
340 id: signature_id(&signature),
341 signature,
342 occurrences,
343 share_of_failures: occurrences as f64 / total_failures as f64,
344 members,
345 }
346 })
347 .collect();
348 clusters.sort_by(|left, right| cluster_sort_key(left).cmp(&cluster_sort_key(right)));
349
350 let result = FailureClusterReport {
351 format_version: FAILURE_CLUSTER_REPORT_FORMAT_VERSION,
352 experiment: source.name.clone(),
353 dataset_name: source.dataset_name.clone(),
354 dataset_version: source.dataset_version.clone(),
355 total_runs: source.summary.runs,
356 total_failures,
357 clusters,
358 };
359 result.validate()?;
360 Ok(result)
361}
362
363fn evidence_refs(run: &CaseRunReport) -> Vec<FailureEvidenceRef> {
364 let mut evidence = Vec::new();
365 match &run.status {
366 RunStatus::Done => {}
367 RunStatus::Interrupted => evidence.push(FailureEvidenceRef::Interrupted),
368 RunStatus::Failed { error } => evidence.push(FailureEvidenceRef::Execution {
369 category: classify_execution_error(error),
370 fingerprint: text_fingerprint("execution", error),
371 }),
372 }
373 for assertion in run.assertions.iter().filter(|assertion| !assertion.passed) {
374 evidence.push(FailureEvidenceRef::Assertion {
375 assertion: assertion.assertion.clone(),
376 expected_fingerprint: value_fingerprint(&assertion.expected),
377 });
378 }
379 if let Some(judge) = run.judge.as_ref().filter(|judge| !judge.passed) {
380 evidence.push(FailureEvidenceRef::Judge {
381 rationale_fingerprint: text_fingerprint("judge", &judge.rationale),
382 });
383 }
384 evidence
385}
386
387fn signature_from_evidence(evidence: &[FailureEvidenceRef]) -> Result<FailureSignature> {
388 let mut primary: Option<FailureCause> = None;
389 let mut failed_assertions = Vec::new();
390 let mut judge_fingerprint = None;
391 for item in evidence {
392 match item {
393 FailureEvidenceRef::Execution {
394 category,
395 fingerprint,
396 } => set_primary(
397 &mut primary,
398 FailureCause::Execution {
399 category: *category,
400 fingerprint: fingerprint.clone(),
401 },
402 )?,
403 FailureEvidenceRef::Interrupted => {
404 set_primary(&mut primary, FailureCause::Interrupted)?;
405 }
406 FailureEvidenceRef::Assertion {
407 assertion,
408 expected_fingerprint,
409 } => failed_assertions.push(AssertionFailureKey {
410 assertion: assertion.clone(),
411 expected_fingerprint: expected_fingerprint.clone(),
412 }),
413 FailureEvidenceRef::Judge {
414 rationale_fingerprint,
415 } => {
416 if judge_fingerprint
417 .replace(rationale_fingerprint.clone())
418 .is_some()
419 {
420 return clustering_error("member evidence contains multiple judge failures");
421 }
422 }
423 }
424 }
425 failed_assertions.sort();
426 failed_assertions.dedup();
427 let (termination, cause) = match primary {
428 Some(FailureCause::Execution {
429 category,
430 fingerprint,
431 }) => (
432 FailureTermination::Failed,
433 FailureCause::Execution {
434 category,
435 fingerprint,
436 },
437 ),
438 Some(FailureCause::Interrupted) => {
439 (FailureTermination::Interrupted, FailureCause::Interrupted)
440 }
441 Some(FailureCause::Evaluation { .. }) => {
442 return clustering_error("evaluation cannot be primary member evidence")
443 }
444 None if !failed_assertions.is_empty() || judge_fingerprint.is_some() => (
445 FailureTermination::Done,
446 FailureCause::Evaluation {
447 fingerprint: evaluation_fingerprint(
448 &failed_assertions,
449 judge_fingerprint.as_deref(),
450 ),
451 },
452 ),
453 None => return clustering_error("failed member has no failure evidence"),
454 };
455 Ok(FailureSignature {
456 termination,
457 cause,
458 failed_assertions,
459 judge_fingerprint,
460 })
461}
462
463fn set_primary(target: &mut Option<FailureCause>, cause: FailureCause) -> Result<()> {
464 if target.replace(cause).is_some() {
465 return clustering_error("member evidence contains multiple termination causes");
466 }
467 Ok(())
468}
469
470fn validate_signature(signature: &FailureSignature) -> Result<()> {
471 if !is_sorted_unique(&signature.failed_assertions) {
472 return clustering_error("failed assertion keys must be sorted and unique");
473 }
474 for failure in &signature.failed_assertions {
475 require_non_empty("assertion key", &failure.assertion)?;
476 validate_fingerprint("assertion expectation", &failure.expected_fingerprint)?;
477 }
478 if let Some(fingerprint) = &signature.judge_fingerprint {
479 validate_fingerprint("judge rationale", fingerprint)?;
480 }
481 match (&signature.termination, &signature.cause) {
482 (FailureTermination::Done, FailureCause::Evaluation { fingerprint }) => {
483 validate_fingerprint("evaluation cause", fingerprint)?;
484 let expected = evaluation_fingerprint(
485 &signature.failed_assertions,
486 signature.judge_fingerprint.as_deref(),
487 );
488 if fingerprint != &expected {
489 return clustering_error("evaluation cause fingerprint is not canonical");
490 }
491 if signature.failed_assertions.is_empty() && signature.judge_fingerprint.is_none() {
492 return clustering_error("completed failure needs assertion or judge evidence");
493 }
494 }
495 (FailureTermination::Failed, FailureCause::Execution { fingerprint, .. }) => {
496 validate_fingerprint("execution cause", fingerprint)?
497 }
498 (FailureTermination::Interrupted, FailureCause::Interrupted) => {}
499 _ => return clustering_error("termination and failure cause do not agree"),
500 }
501 Ok(())
502}
503
504fn classify_execution_error(error: &str) -> ExecutionFailureCategory {
505 let lower = error.to_ascii_lowercase();
506 if contains_any(&lower, &["unauthorized", "authentication", "invalid token"])
507 || contains_status_code(&lower, "401")
508 {
509 ExecutionFailureCategory::Authentication
510 } else if contains_any(&lower, &["forbidden", "permission denied", "not permitted"])
511 || contains_status_code(&lower, "403")
512 {
513 ExecutionFailureCategory::Authorization
514 } else if contains_any(&lower, &["timeout", "timed out", "deadline exceeded"]) {
515 ExecutionFailureCategory::Timeout
516 } else if contains_any(&lower, &["rate limit", "too many requests", "quota"])
517 || contains_status_code(&lower, "429")
518 {
519 ExecutionFailureCategory::RateLimit
520 } else if contains_any(&lower, &["cancelled", "canceled"]) {
521 ExecutionFailureCategory::Cancelled
522 } else if contains_any(
523 &lower,
524 &["connection", "connect", "dns", "network", "transport"],
525 ) {
526 ExecutionFailureCategory::Transport
527 } else if contains_any(&lower, &["unavailable", "server error"])
528 || ["500", "502", "503", "504"]
529 .iter()
530 .any(|code| contains_status_code(&lower, code))
531 {
532 ExecutionFailureCategory::Unavailable
533 } else if contains_any(
534 &lower,
535 &[
536 "schema",
537 "deserialize",
538 "invalid json",
539 "parse error",
540 "validation",
541 ],
542 ) {
543 ExecutionFailureCategory::InvalidData
544 } else {
545 ExecutionFailureCategory::Unknown
546 }
547}
548
549fn contains_any(value: &str, needles: &[&str]) -> bool {
550 needles.iter().any(|needle| value.contains(needle))
551}
552
553fn contains_status_code(value: &str, code: &str) -> bool {
554 value
555 .split(|character: char| !character.is_ascii_alphanumeric())
556 .any(|token| token == code)
557}
558
559fn normalized_text(value: &str) -> String {
560 scrub_volatile_ids(value)
561 .split_whitespace()
562 .map(|raw| {
563 let token = raw.trim_matches(|character: char| {
564 !character.is_ascii_alphanumeric() && character != '-'
565 });
566 let mut normalized = String::new();
567 let mut replacing_digits = false;
568 for character in token.chars().flat_map(char::to_lowercase) {
569 if character.is_ascii_digit() {
570 if !replacing_digits {
571 normalized.push('#');
572 replacing_digits = true;
573 }
574 } else {
575 replacing_digits = false;
576 if character.is_alphanumeric() || matches!(character, '_' | '-' | '.' | '/') {
577 normalized.push(character);
578 }
579 }
580 }
581 normalized
582 })
583 .filter(|token| !token.is_empty())
584 .collect::<Vec<_>>()
585 .join(" ")
586}
587
588fn scrub_volatile_ids(value: &str) -> String {
589 let characters: Vec<_> = value.chars().collect();
590 let mut scrubbed = String::with_capacity(value.len());
591 let mut index = 0;
592
593 while index < characters.len() {
594 if is_uuid_at(&characters, index) {
595 scrubbed.push_str(" volatileid ");
596 index += 36;
597 continue;
598 }
599
600 if characters[index].is_ascii_hexdigit()
601 && (index == 0 || !characters[index - 1].is_ascii_alphanumeric())
602 {
603 let end = characters[index..]
604 .iter()
605 .position(|character| !character.is_ascii_hexdigit())
606 .map_or(characters.len(), |offset| index + offset);
607 let candidate = &characters[index..end];
608 if candidate.len() >= 8
609 && candidate.iter().any(|character| character.is_ascii_digit())
610 && (end == characters.len() || !characters[end].is_ascii_alphanumeric())
611 {
612 scrubbed.push_str(" volatileid ");
613 index = end;
614 continue;
615 }
616 }
617
618 scrubbed.push(characters[index]);
619 index += 1;
620 }
621
622 scrubbed
623}
624
625fn is_uuid_at(characters: &[char], start: usize) -> bool {
626 const UUID_LENGTH: usize = 36;
627 const HYPHENS: [usize; 4] = [8, 13, 18, 23];
628
629 let Some(candidate) = characters.get(start..start + UUID_LENGTH) else {
630 return false;
631 };
632 if (start > 0 && characters[start - 1].is_ascii_alphanumeric())
633 || characters
634 .get(start + UUID_LENGTH)
635 .is_some_and(|character| character.is_ascii_alphanumeric())
636 {
637 return false;
638 }
639
640 candidate.iter().enumerate().all(|(offset, character)| {
641 if HYPHENS.contains(&offset) {
642 *character == '-'
643 } else {
644 character.is_ascii_hexdigit()
645 }
646 })
647}
648
649fn text_fingerprint(domain: &str, value: &str) -> String {
650 fingerprint(domain, normalized_text(value).as_bytes())
651}
652
653fn value_fingerprint(value: &Value) -> String {
654 let mut canonical = Vec::new();
655 append_canonical_json(&mut canonical, value);
656 fingerprint("assertion-expected", &canonical)
657}
658
659fn append_canonical_json(bytes: &mut Vec<u8>, value: &Value) {
660 match value {
661 Value::Null => bytes.push(0),
662 Value::Bool(value) => {
663 bytes.push(1);
664 bytes.push(u8::from(*value));
665 }
666 Value::Number(value) => {
667 bytes.push(2);
668 append_bytes(bytes, value.to_string().as_bytes());
669 }
670 Value::String(value) => {
671 bytes.push(3);
672 append_bytes(bytes, value.as_bytes());
673 }
674 Value::Array(values) => {
675 bytes.push(4);
676 append_u64(bytes, values.len() as u64);
677 for value in values {
678 append_canonical_json(bytes, value);
679 }
680 }
681 Value::Object(values) => {
682 bytes.push(5);
683 append_u64(bytes, values.len() as u64);
684 let mut entries: Vec<_> = values.iter().collect();
685 entries.sort_unstable_by_key(|(key, _)| *key);
686 for (key, value) in entries {
687 append_bytes(bytes, key.as_bytes());
688 append_canonical_json(bytes, value);
689 }
690 }
691 }
692}
693
694fn evaluation_fingerprint(
695 assertions: &[AssertionFailureKey],
696 judge_fingerprint: Option<&str>,
697) -> String {
698 let mut bytes = Vec::new();
699 append_u64(&mut bytes, assertions.len() as u64);
700 for assertion in assertions {
701 append_bytes(&mut bytes, assertion.assertion.as_bytes());
702 append_bytes(&mut bytes, assertion.expected_fingerprint.as_bytes());
703 }
704 match judge_fingerprint {
705 Some(fingerprint) => {
706 bytes.push(1);
707 append_bytes(&mut bytes, fingerprint.as_bytes());
708 }
709 None => bytes.push(0),
710 }
711 fingerprint("evaluation", &bytes)
712}
713
714fn fingerprint(domain: &str, value: &[u8]) -> String {
715 let mut hasher = Sha256::new();
716 hasher.update(b"rusty.failure-fingerprint.v1\0");
717 hasher.update((domain.len() as u64).to_be_bytes());
718 hasher.update(domain.as_bytes());
719 hasher.update((value.len() as u64).to_be_bytes());
720 hasher.update(value);
721 format_digest("fp_", &hasher.finalize())
722}
723
724fn signature_id(signature: &FailureSignature) -> String {
725 let encoded = serde_json::to_vec(signature).expect("serializing a signature cannot fail");
726 let mut hasher = Sha256::new();
727 hasher.update(b"rusty.failure-cluster.v1\0");
728 hasher.update((encoded.len() as u64).to_be_bytes());
729 hasher.update(encoded);
730 format_digest("fc_", &hasher.finalize())
731}
732
733fn format_digest(prefix: &str, digest: &[u8]) -> String {
734 let mut result = String::from(prefix);
735 for byte in &digest[..16] {
736 write!(&mut result, "{byte:02x}").expect("writing to a String cannot fail");
737 }
738 result
739}
740
741fn append_u64(target: &mut Vec<u8>, value: u64) {
742 target.extend_from_slice(&value.to_be_bytes());
743}
744
745fn append_bytes(target: &mut Vec<u8>, value: &[u8]) {
746 append_u64(target, value.len() as u64);
747 target.extend_from_slice(value);
748}
749
750fn validate_fingerprint(label: &str, fingerprint: &str) -> Result<()> {
751 if fingerprint.len() != 35
752 || !fingerprint.starts_with("fp_")
753 || !fingerprint[3..]
754 .bytes()
755 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
756 {
757 return clustering_error(format!("{label} fingerprint is malformed"));
758 }
759 Ok(())
760}
761
762fn member_key(member: &FailureOccurrence) -> (&str, usize) {
763 (&member.case_id, member.repetition)
764}
765
766fn cluster_sort_key(cluster: &FailureCluster) -> (std::cmp::Reverse<usize>, &FailureSignature) {
767 (std::cmp::Reverse(cluster.occurrences), &cluster.signature)
768}
769
770fn is_sorted_unique<T: Ord>(values: &[T]) -> bool {
771 values.windows(2).all(|pair| pair[0] < pair[1])
772}
773
774fn require_non_empty(label: &str, value: &str) -> Result<()> {
775 if value.trim().is_empty() {
776 return clustering_error(format!("{label} must not be empty"));
777 }
778 Ok(())
779}
780
781fn same_float(actual: f64, expected: f64) -> bool {
782 if !actual.is_finite() || !expected.is_finite() {
783 return false;
784 }
785 if actual == expected {
786 return true;
787 }
788 let scale = actual.abs().max(expected.abs());
789 (actual - expected).abs() <= 16.0 * f64::EPSILON * scale
790}
791
792fn reports_match(actual: &FailureClusterReport, expected: &FailureClusterReport) -> bool {
793 actual.format_version == expected.format_version
794 && actual.experiment == expected.experiment
795 && actual.dataset_name == expected.dataset_name
796 && actual.dataset_version == expected.dataset_version
797 && actual.total_runs == expected.total_runs
798 && actual.total_failures == expected.total_failures
799 && actual.clusters.len() == expected.clusters.len()
800 && actual
801 .clusters
802 .iter()
803 .zip(&expected.clusters)
804 .all(|(actual, expected)| {
805 actual.id == expected.id
806 && actual.signature == expected.signature
807 && actual.occurrences == expected.occurrences
808 && same_float(actual.share_of_failures, expected.share_of_failures)
809 && actual.members == expected.members
810 })
811}
812
813fn clustering_error<T>(message: impl Into<String>) -> Result<T> {
814 Err(EvalError::Clustering(message.into()))
815}
816
817#[derive(Deserialize)]
818struct FormatHeader {
819 format_version: u64,
820}
821
822#[derive(Deserialize)]
823#[serde(deny_unknown_fields)]
824struct AssertionFailureKeyWire {
825 assertion: String,
826 expected_fingerprint: String,
827}
828
829impl From<AssertionFailureKeyWire> for AssertionFailureKey {
830 fn from(value: AssertionFailureKeyWire) -> Self {
831 Self {
832 assertion: value.assertion,
833 expected_fingerprint: value.expected_fingerprint,
834 }
835 }
836}
837
838#[derive(Deserialize)]
839#[serde(tag = "cause", rename_all = "snake_case", deny_unknown_fields)]
840enum FailureCauseWire {
841 Evaluation {
842 fingerprint: String,
843 },
844 Interrupted,
845 Execution {
846 category: ExecutionFailureCategory,
847 fingerprint: String,
848 },
849}
850
851impl From<FailureCauseWire> for FailureCause {
852 fn from(value: FailureCauseWire) -> Self {
853 match value {
854 FailureCauseWire::Evaluation { fingerprint } => Self::Evaluation { fingerprint },
855 FailureCauseWire::Interrupted => Self::Interrupted,
856 FailureCauseWire::Execution {
857 category,
858 fingerprint,
859 } => Self::Execution {
860 category,
861 fingerprint,
862 },
863 }
864 }
865}
866
867#[derive(Deserialize)]
868#[serde(deny_unknown_fields)]
869struct FailureSignatureWire {
870 termination: FailureTermination,
871 cause: FailureCauseWire,
872 failed_assertions: Vec<AssertionFailureKeyWire>,
873 judge_fingerprint: Option<String>,
874}
875
876impl From<FailureSignatureWire> for FailureSignature {
877 fn from(value: FailureSignatureWire) -> Self {
878 Self {
879 termination: value.termination,
880 cause: value.cause.into(),
881 failed_assertions: value
882 .failed_assertions
883 .into_iter()
884 .map(Into::into)
885 .collect(),
886 judge_fingerprint: value.judge_fingerprint,
887 }
888 }
889}
890
891#[derive(Deserialize)]
892#[serde(tag = "evidence", rename_all = "snake_case", deny_unknown_fields)]
893enum FailureEvidenceRefWire {
894 Execution {
895 category: ExecutionFailureCategory,
896 fingerprint: String,
897 },
898 Interrupted,
899 Assertion {
900 assertion: String,
901 expected_fingerprint: String,
902 },
903 Judge {
904 rationale_fingerprint: String,
905 },
906}
907
908impl From<FailureEvidenceRefWire> for FailureEvidenceRef {
909 fn from(value: FailureEvidenceRefWire) -> Self {
910 match value {
911 FailureEvidenceRefWire::Execution {
912 category,
913 fingerprint,
914 } => Self::Execution {
915 category,
916 fingerprint,
917 },
918 FailureEvidenceRefWire::Interrupted => Self::Interrupted,
919 FailureEvidenceRefWire::Assertion {
920 assertion,
921 expected_fingerprint,
922 } => Self::Assertion {
923 assertion,
924 expected_fingerprint,
925 },
926 FailureEvidenceRefWire::Judge {
927 rationale_fingerprint,
928 } => Self::Judge {
929 rationale_fingerprint,
930 },
931 }
932 }
933}
934
935#[derive(Deserialize)]
936#[serde(deny_unknown_fields)]
937struct FailureOccurrenceWire {
938 case_id: String,
939 repetition: usize,
940 tags: Vec<String>,
941 evidence: Vec<FailureEvidenceRefWire>,
942}
943
944impl From<FailureOccurrenceWire> for FailureOccurrence {
945 fn from(value: FailureOccurrenceWire) -> Self {
946 Self {
947 case_id: value.case_id,
948 repetition: value.repetition,
949 tags: value.tags,
950 evidence: value.evidence.into_iter().map(Into::into).collect(),
951 }
952 }
953}
954
955#[derive(Deserialize)]
956#[serde(deny_unknown_fields)]
957struct FailureClusterWire {
958 id: String,
959 signature: FailureSignatureWire,
960 occurrences: usize,
961 share_of_failures: f64,
962 members: Vec<FailureOccurrenceWire>,
963}
964
965impl From<FailureClusterWire> for FailureCluster {
966 fn from(value: FailureClusterWire) -> Self {
967 Self {
968 id: value.id,
969 signature: value.signature.into(),
970 occurrences: value.occurrences,
971 share_of_failures: value.share_of_failures,
972 members: value.members.into_iter().map(Into::into).collect(),
973 }
974 }
975}
976
977#[derive(Deserialize)]
978#[serde(deny_unknown_fields)]
979struct FailureClusterReportWire {
980 format_version: u64,
981 experiment: String,
982 dataset_name: String,
983 dataset_version: String,
984 total_runs: usize,
985 total_failures: usize,
986 clusters: Vec<FailureClusterWire>,
987}
988
989impl FailureClusterReportWire {
990 fn deserialize_report(text: &str) -> Result<FailureClusterReport> {
991 let wire: Self = serde_json::from_str(text)?;
992 Ok(FailureClusterReport {
993 format_version: wire.format_version,
994 experiment: wire.experiment,
995 dataset_name: wire.dataset_name,
996 dataset_version: wire.dataset_version,
997 total_runs: wire.total_runs,
998 total_failures: wire.total_failures,
999 clusters: wire.clusters.into_iter().map(Into::into).collect(),
1000 })
1001 }
1002}