Skip to main content

rusty_eval/
clustering.rs

1//! Deterministic clustering of failed experiment runs.
2//!
3//! [`cluster_failures`] groups failures by normalized, machine-verifiable
4//! causes. Raw executor errors, assertion observations, and judge rationales
5//! remain in the source report; cluster artifacts carry only source
6//! coordinates, categories, and content fingerprints. This keeps grouping
7//! stable across volatile request ids without copying sensitive evidence.
8
9use std::collections::{BTreeMap, BTreeSet};
10use std::fmt::Write as _;
11
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14use sha2::{Digest, Sha256};
15
16use crate::error::{EvalError, Result};
17use crate::evidence::RunStatus;
18use crate::experiment::{CaseRunReport, ExperimentReport};
19
20/// Wire version for [`FailureClusterReport`].
21pub const FAILURE_CLUSTER_REPORT_FORMAT_VERSION: u64 = 1;
22
23/// Coarse execution outcome used in a failure signature.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
25#[serde(rename_all = "snake_case")]
26pub enum FailureTermination {
27    /// Execution completed, but an assertion or judge rejected the result.
28    Done,
29    /// Execution suspended before producing a terminal result.
30    Interrupted,
31    /// Execution returned an error.
32    Failed,
33}
34
35/// Normalized executor-error category.
36#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
37#[serde(rename_all = "snake_case")]
38pub enum ExecutionFailureCategory {
39    /// Missing, rejected, or expired authentication material.
40    Authentication,
41    /// Authenticated principal lacks permission.
42    Authorization,
43    /// Deadline or timeout exhaustion.
44    Timeout,
45    /// Provider or tenant rate/quota limit.
46    RateLimit,
47    /// Connection, DNS, or other transport failure.
48    Transport,
49    /// Upstream service is unavailable or returned a server error.
50    Unavailable,
51    /// Invalid schema, JSON, parsing, or validation input.
52    InvalidData,
53    /// Explicit cancellation.
54    Cancelled,
55    /// No stable category matched.
56    Unknown,
57}
58
59/// Normalized primary cause of a failed run.
60#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
61#[serde(tag = "cause", rename_all = "snake_case")]
62pub enum FailureCause {
63    /// A completed run failed evaluation.
64    Evaluation {
65        /// Fingerprint of assertion configurations and judge rationale.
66        fingerprint: String,
67    },
68    /// A run suspended before completion.
69    Interrupted,
70    /// A normalized executor error.
71    Execution {
72        /// Actionable error family.
73        category: ExecutionFailureCategory,
74        /// Fingerprint of normalized error text with volatile ids removed.
75        fingerprint: String,
76    },
77}
78
79/// One failed assertion configuration within a cluster signature.
80#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
81pub struct AssertionFailureKey {
82    /// Stable assertion report key.
83    pub assertion: String,
84    /// Fingerprint of the assertion's expected/configuration value.
85    pub expected_fingerprint: String,
86}
87
88/// The stable, explainable dimensions that define one failure cluster.
89#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
90pub struct FailureSignature {
91    /// How runs in this cluster terminated.
92    pub termination: FailureTermination,
93    /// Primary normalized failure cause.
94    pub cause: FailureCause,
95    /// Sorted, deduplicated failed assertion configurations.
96    pub failed_assertions: Vec<AssertionFailureKey>,
97    /// Fingerprint of a rejecting judge rationale, when present.
98    pub judge_fingerprint: Option<String>,
99}
100
101/// Safe structured evidence retained in a cluster artifact.
102#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
103#[serde(tag = "evidence", rename_all = "snake_case")]
104pub enum FailureEvidenceRef {
105    /// Normalized executor-error evidence.
106    Execution {
107        /// Actionable error family.
108        category: ExecutionFailureCategory,
109        /// Fingerprint of normalized error text.
110        fingerprint: String,
111    },
112    /// The source run suspended.
113    Interrupted,
114    /// Failed deterministic assertion configuration.
115    Assertion {
116        /// Stable assertion report key.
117        assertion: String,
118        /// Fingerprint of expected/configuration value; observed data is not copied.
119        expected_fingerprint: String,
120    },
121    /// Rejecting judge rationale fingerprint; rationale text is not copied.
122    Judge {
123        /// Fingerprint of normalized rationale text.
124        rationale_fingerprint: String,
125    },
126}
127
128/// One failed run linked back to its experiment evidence.
129#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
130pub struct FailureOccurrence {
131    /// Dataset case id.
132    pub case_id: String,
133    /// Zero-based run repetition.
134    pub repetition: usize,
135    /// Sorted case tags for filtering and ownership routing.
136    pub tags: Vec<String>,
137    /// Safe evidence references; raw values remain in the source report.
138    pub evidence: Vec<FailureEvidenceRef>,
139}
140
141/// A group of failed runs with the same [`FailureSignature`].
142#[derive(Debug, Clone, PartialEq, Serialize)]
143pub struct FailureCluster {
144    /// Stable SHA-256-derived identity of the signature.
145    pub id: String,
146    /// Machine-verifiable grouping dimensions.
147    pub signature: FailureSignature,
148    /// Number of members in this cluster.
149    pub occurrences: usize,
150    /// `occurrences / total_failures`.
151    pub share_of_failures: f64,
152    /// Members sorted by case id and repetition.
153    pub members: Vec<FailureOccurrence>,
154}
155
156/// Ranked failure clusters for one experiment report.
157#[derive(Debug, Clone, PartialEq, Serialize)]
158pub struct FailureClusterReport {
159    /// Report schema version.
160    pub format_version: u64,
161    /// Source experiment identity.
162    pub experiment: String,
163    /// Source dataset name.
164    pub dataset_name: String,
165    /// Source dataset version.
166    pub dataset_version: String,
167    /// Number of source case runs.
168    pub total_runs: usize,
169    /// Number of failed source case runs.
170    pub total_failures: usize,
171    /// Clusters ordered by descending frequency, then signature.
172    pub clusters: Vec<FailureCluster>,
173}
174
175impl FailureClusterReport {
176    /// Serialize a validated clustering artifact as pretty JSON.
177    pub fn to_json(&self) -> Result<String> {
178        self.validate()?;
179        Ok(serde_json::to_string_pretty(self)?)
180    }
181
182    /// Parse an artifact and recompute its canonical value from source evidence.
183    ///
184    /// The returned value is always the recomputed report, never untrusted
185    /// caller-supplied derived fields. Raw deserialization is intentionally not
186    /// implemented for the public artifact types.
187    pub fn from_json(text: &str, source: &ExperimentReport) -> Result<Self> {
188        let header: FormatHeader = serde_json::from_str(text)?;
189        if header.format_version != FAILURE_CLUSTER_REPORT_FORMAT_VERSION {
190            return clustering_error(format!(
191                "unsupported failure cluster format version: found {}, this build supports {}",
192                header.format_version, FAILURE_CLUSTER_REPORT_FORMAT_VERSION
193            ));
194        }
195        let supplied = FailureClusterReportWire::deserialize_report(text)?;
196        supplied.validate()?;
197        let expected = cluster_failures(source)?;
198        if !reports_match(&supplied, &expected) {
199            return clustering_error("failure cluster report does not match its source evidence");
200        }
201        Ok(expected)
202    }
203
204    fn validate(&self) -> Result<()> {
205        if self.format_version != FAILURE_CLUSTER_REPORT_FORMAT_VERSION {
206            return clustering_error(format!(
207                "unsupported failure cluster format version: found {}, this build supports {}",
208                self.format_version, FAILURE_CLUSTER_REPORT_FORMAT_VERSION
209            ));
210        }
211        require_non_empty("experiment", &self.experiment)?;
212        require_non_empty("dataset name", &self.dataset_name)?;
213        require_non_empty("dataset version", &self.dataset_version)?;
214        if self.total_failures > self.total_runs {
215            return clustering_error("total failures cannot exceed total runs");
216        }
217        if (self.total_failures == 0) != self.clusters.is_empty() {
218            return clustering_error("clusters must be empty exactly when there are no failures");
219        }
220
221        let mut signatures = BTreeSet::new();
222        let mut ids = BTreeSet::new();
223        let mut occurrences = BTreeSet::new();
224        let mut counted_failures = 0_usize;
225        for cluster in &self.clusters {
226            validate_signature(&cluster.signature)?;
227            let expected_id = signature_id(&cluster.signature);
228            if cluster.id != expected_id {
229                return clustering_error(format!(
230                    "cluster id `{}` does not match its signature",
231                    cluster.id
232                ));
233            }
234            if !ids.insert(cluster.id.as_str()) || !signatures.insert(&cluster.signature) {
235                return clustering_error("cluster ids and signatures must be unique");
236            }
237            if cluster.occurrences == 0 || cluster.occurrences != cluster.members.len() {
238                return clustering_error(format!(
239                    "cluster `{}` occurrence count does not match its members",
240                    cluster.id
241                ));
242            }
243            counted_failures = counted_failures
244                .checked_add(cluster.occurrences)
245                .ok_or_else(|| EvalError::Clustering("failure count overflows usize".to_owned()))?;
246            let expected_share = cluster.occurrences as f64 / self.total_failures as f64;
247            if !same_float(cluster.share_of_failures, expected_share) {
248                return clustering_error(format!(
249                    "cluster `{}` share does not match its occurrence count",
250                    cluster.id
251                ));
252            }
253            if !cluster
254                .members
255                .windows(2)
256                .all(|pair| member_key(&pair[0]) < member_key(&pair[1]))
257            {
258                return clustering_error(format!(
259                    "cluster `{}` members are not in canonical order",
260                    cluster.id
261                ));
262            }
263            for member in &cluster.members {
264                require_non_empty("failure case id", &member.case_id)?;
265                if member.evidence.is_empty()
266                    || !is_sorted_unique(&member.evidence)
267                    || !is_sorted_unique(&member.tags)
268                {
269                    return clustering_error(format!(
270                        "cluster `{}` contains non-canonical member evidence",
271                        cluster.id
272                    ));
273                }
274                let member_signature = signature_from_evidence(&member.evidence)?;
275                if member_signature != cluster.signature {
276                    return clustering_error(format!(
277                        "cluster `{}` member evidence does not match its signature",
278                        cluster.id
279                    ));
280                }
281                if !occurrences.insert((member.case_id.as_str(), member.repetition)) {
282                    return clustering_error(format!(
283                        "failure occurrence `{}#{}` appears more than once",
284                        member.case_id, member.repetition
285                    ));
286                }
287            }
288        }
289        if counted_failures != self.total_failures {
290            return clustering_error("cluster occurrence counts do not equal total failures");
291        }
292        if !self
293            .clusters
294            .windows(2)
295            .all(|pair| cluster_sort_key(&pair[0]) < cluster_sort_key(&pair[1]))
296        {
297            return clustering_error("clusters are not in canonical rank order");
298        }
299        Ok(())
300    }
301}
302
303/// Group every failed run in an experiment by deterministic root-cause shape.
304pub fn cluster_failures(source: &ExperimentReport) -> Result<FailureClusterReport> {
305    crate::gate::validate_report(source)
306        .map_err(|error| EvalError::Clustering(format!("source report is invalid: {error}")))?;
307
308    let mut grouped: BTreeMap<FailureSignature, Vec<FailureOccurrence>> = BTreeMap::new();
309    for case in &source.cases {
310        for run in &case.runs {
311            if run.passed {
312                continue;
313            }
314            let mut evidence = evidence_refs(run);
315            evidence.sort();
316            evidence.dedup();
317            let signature = signature_from_evidence(&evidence)?;
318            let mut tags = case.tags.clone();
319            tags.sort();
320            tags.dedup();
321            grouped
322                .entry(signature)
323                .or_default()
324                .push(FailureOccurrence {
325                    case_id: case.case_id.clone(),
326                    repetition: run.repetition,
327                    tags,
328                    evidence,
329                });
330        }
331    }
332
333    let total_failures = grouped.values().map(Vec::len).sum();
334    let mut clusters: Vec<_> = grouped
335        .into_iter()
336        .map(|(signature, mut members)| {
337            members.sort_by(|left, right| member_key(left).cmp(&member_key(right)));
338            let occurrences = members.len();
339            FailureCluster {
340                id: signature_id(&signature),
341                signature,
342                occurrences,
343                share_of_failures: occurrences as f64 / total_failures as f64,
344                members,
345            }
346        })
347        .collect();
348    clusters.sort_by(|left, right| cluster_sort_key(left).cmp(&cluster_sort_key(right)));
349
350    let result = FailureClusterReport {
351        format_version: FAILURE_CLUSTER_REPORT_FORMAT_VERSION,
352        experiment: source.name.clone(),
353        dataset_name: source.dataset_name.clone(),
354        dataset_version: source.dataset_version.clone(),
355        total_runs: source.summary.runs,
356        total_failures,
357        clusters,
358    };
359    result.validate()?;
360    Ok(result)
361}
362
363fn evidence_refs(run: &CaseRunReport) -> Vec<FailureEvidenceRef> {
364    let mut evidence = Vec::new();
365    match &run.status {
366        RunStatus::Done => {}
367        RunStatus::Interrupted => evidence.push(FailureEvidenceRef::Interrupted),
368        RunStatus::Failed { error } => evidence.push(FailureEvidenceRef::Execution {
369            category: classify_execution_error(error),
370            fingerprint: text_fingerprint("execution", error),
371        }),
372    }
373    for assertion in run.assertions.iter().filter(|assertion| !assertion.passed) {
374        evidence.push(FailureEvidenceRef::Assertion {
375            assertion: assertion.assertion.clone(),
376            expected_fingerprint: value_fingerprint(&assertion.expected),
377        });
378    }
379    if let Some(judge) = run.judge.as_ref().filter(|judge| !judge.passed) {
380        evidence.push(FailureEvidenceRef::Judge {
381            rationale_fingerprint: text_fingerprint("judge", &judge.rationale),
382        });
383    }
384    evidence
385}
386
387fn signature_from_evidence(evidence: &[FailureEvidenceRef]) -> Result<FailureSignature> {
388    let mut primary: Option<FailureCause> = None;
389    let mut failed_assertions = Vec::new();
390    let mut judge_fingerprint = None;
391    for item in evidence {
392        match item {
393            FailureEvidenceRef::Execution {
394                category,
395                fingerprint,
396            } => set_primary(
397                &mut primary,
398                FailureCause::Execution {
399                    category: *category,
400                    fingerprint: fingerprint.clone(),
401                },
402            )?,
403            FailureEvidenceRef::Interrupted => {
404                set_primary(&mut primary, FailureCause::Interrupted)?;
405            }
406            FailureEvidenceRef::Assertion {
407                assertion,
408                expected_fingerprint,
409            } => failed_assertions.push(AssertionFailureKey {
410                assertion: assertion.clone(),
411                expected_fingerprint: expected_fingerprint.clone(),
412            }),
413            FailureEvidenceRef::Judge {
414                rationale_fingerprint,
415            } => {
416                if judge_fingerprint
417                    .replace(rationale_fingerprint.clone())
418                    .is_some()
419                {
420                    return clustering_error("member evidence contains multiple judge failures");
421                }
422            }
423        }
424    }
425    failed_assertions.sort();
426    failed_assertions.dedup();
427    let (termination, cause) = match primary {
428        Some(FailureCause::Execution {
429            category,
430            fingerprint,
431        }) => (
432            FailureTermination::Failed,
433            FailureCause::Execution {
434                category,
435                fingerprint,
436            },
437        ),
438        Some(FailureCause::Interrupted) => {
439            (FailureTermination::Interrupted, FailureCause::Interrupted)
440        }
441        Some(FailureCause::Evaluation { .. }) => {
442            return clustering_error("evaluation cannot be primary member evidence")
443        }
444        None if !failed_assertions.is_empty() || judge_fingerprint.is_some() => (
445            FailureTermination::Done,
446            FailureCause::Evaluation {
447                fingerprint: evaluation_fingerprint(
448                    &failed_assertions,
449                    judge_fingerprint.as_deref(),
450                ),
451            },
452        ),
453        None => return clustering_error("failed member has no failure evidence"),
454    };
455    Ok(FailureSignature {
456        termination,
457        cause,
458        failed_assertions,
459        judge_fingerprint,
460    })
461}
462
463fn set_primary(target: &mut Option<FailureCause>, cause: FailureCause) -> Result<()> {
464    if target.replace(cause).is_some() {
465        return clustering_error("member evidence contains multiple termination causes");
466    }
467    Ok(())
468}
469
470fn validate_signature(signature: &FailureSignature) -> Result<()> {
471    if !is_sorted_unique(&signature.failed_assertions) {
472        return clustering_error("failed assertion keys must be sorted and unique");
473    }
474    for failure in &signature.failed_assertions {
475        require_non_empty("assertion key", &failure.assertion)?;
476        validate_fingerprint("assertion expectation", &failure.expected_fingerprint)?;
477    }
478    if let Some(fingerprint) = &signature.judge_fingerprint {
479        validate_fingerprint("judge rationale", fingerprint)?;
480    }
481    match (&signature.termination, &signature.cause) {
482        (FailureTermination::Done, FailureCause::Evaluation { fingerprint }) => {
483            validate_fingerprint("evaluation cause", fingerprint)?;
484            let expected = evaluation_fingerprint(
485                &signature.failed_assertions,
486                signature.judge_fingerprint.as_deref(),
487            );
488            if fingerprint != &expected {
489                return clustering_error("evaluation cause fingerprint is not canonical");
490            }
491            if signature.failed_assertions.is_empty() && signature.judge_fingerprint.is_none() {
492                return clustering_error("completed failure needs assertion or judge evidence");
493            }
494        }
495        (FailureTermination::Failed, FailureCause::Execution { fingerprint, .. }) => {
496            validate_fingerprint("execution cause", fingerprint)?
497        }
498        (FailureTermination::Interrupted, FailureCause::Interrupted) => {}
499        _ => return clustering_error("termination and failure cause do not agree"),
500    }
501    Ok(())
502}
503
504fn classify_execution_error(error: &str) -> ExecutionFailureCategory {
505    let lower = error.to_ascii_lowercase();
506    if contains_any(&lower, &["unauthorized", "authentication", "invalid token"])
507        || contains_status_code(&lower, "401")
508    {
509        ExecutionFailureCategory::Authentication
510    } else if contains_any(&lower, &["forbidden", "permission denied", "not permitted"])
511        || contains_status_code(&lower, "403")
512    {
513        ExecutionFailureCategory::Authorization
514    } else if contains_any(&lower, &["timeout", "timed out", "deadline exceeded"]) {
515        ExecutionFailureCategory::Timeout
516    } else if contains_any(&lower, &["rate limit", "too many requests", "quota"])
517        || contains_status_code(&lower, "429")
518    {
519        ExecutionFailureCategory::RateLimit
520    } else if contains_any(&lower, &["cancelled", "canceled"]) {
521        ExecutionFailureCategory::Cancelled
522    } else if contains_any(
523        &lower,
524        &["connection", "connect", "dns", "network", "transport"],
525    ) {
526        ExecutionFailureCategory::Transport
527    } else if contains_any(&lower, &["unavailable", "server error"])
528        || ["500", "502", "503", "504"]
529            .iter()
530            .any(|code| contains_status_code(&lower, code))
531    {
532        ExecutionFailureCategory::Unavailable
533    } else if contains_any(
534        &lower,
535        &[
536            "schema",
537            "deserialize",
538            "invalid json",
539            "parse error",
540            "validation",
541        ],
542    ) {
543        ExecutionFailureCategory::InvalidData
544    } else {
545        ExecutionFailureCategory::Unknown
546    }
547}
548
549fn contains_any(value: &str, needles: &[&str]) -> bool {
550    needles.iter().any(|needle| value.contains(needle))
551}
552
553fn contains_status_code(value: &str, code: &str) -> bool {
554    value
555        .split(|character: char| !character.is_ascii_alphanumeric())
556        .any(|token| token == code)
557}
558
559fn normalized_text(value: &str) -> String {
560    scrub_volatile_ids(value)
561        .split_whitespace()
562        .map(|raw| {
563            let token = raw.trim_matches(|character: char| {
564                !character.is_ascii_alphanumeric() && character != '-'
565            });
566            let mut normalized = String::new();
567            let mut replacing_digits = false;
568            for character in token.chars().flat_map(char::to_lowercase) {
569                if character.is_ascii_digit() {
570                    if !replacing_digits {
571                        normalized.push('#');
572                        replacing_digits = true;
573                    }
574                } else {
575                    replacing_digits = false;
576                    if character.is_alphanumeric() || matches!(character, '_' | '-' | '.' | '/') {
577                        normalized.push(character);
578                    }
579                }
580            }
581            normalized
582        })
583        .filter(|token| !token.is_empty())
584        .collect::<Vec<_>>()
585        .join(" ")
586}
587
588fn scrub_volatile_ids(value: &str) -> String {
589    let characters: Vec<_> = value.chars().collect();
590    let mut scrubbed = String::with_capacity(value.len());
591    let mut index = 0;
592
593    while index < characters.len() {
594        if is_uuid_at(&characters, index) {
595            scrubbed.push_str(" volatileid ");
596            index += 36;
597            continue;
598        }
599
600        if characters[index].is_ascii_hexdigit()
601            && (index == 0 || !characters[index - 1].is_ascii_alphanumeric())
602        {
603            let end = characters[index..]
604                .iter()
605                .position(|character| !character.is_ascii_hexdigit())
606                .map_or(characters.len(), |offset| index + offset);
607            let candidate = &characters[index..end];
608            if candidate.len() >= 8
609                && candidate.iter().any(|character| character.is_ascii_digit())
610                && (end == characters.len() || !characters[end].is_ascii_alphanumeric())
611            {
612                scrubbed.push_str(" volatileid ");
613                index = end;
614                continue;
615            }
616        }
617
618        scrubbed.push(characters[index]);
619        index += 1;
620    }
621
622    scrubbed
623}
624
625fn is_uuid_at(characters: &[char], start: usize) -> bool {
626    const UUID_LENGTH: usize = 36;
627    const HYPHENS: [usize; 4] = [8, 13, 18, 23];
628
629    let Some(candidate) = characters.get(start..start + UUID_LENGTH) else {
630        return false;
631    };
632    if (start > 0 && characters[start - 1].is_ascii_alphanumeric())
633        || characters
634            .get(start + UUID_LENGTH)
635            .is_some_and(|character| character.is_ascii_alphanumeric())
636    {
637        return false;
638    }
639
640    candidate.iter().enumerate().all(|(offset, character)| {
641        if HYPHENS.contains(&offset) {
642            *character == '-'
643        } else {
644            character.is_ascii_hexdigit()
645        }
646    })
647}
648
649fn text_fingerprint(domain: &str, value: &str) -> String {
650    fingerprint(domain, normalized_text(value).as_bytes())
651}
652
653fn value_fingerprint(value: &Value) -> String {
654    let mut canonical = Vec::new();
655    append_canonical_json(&mut canonical, value);
656    fingerprint("assertion-expected", &canonical)
657}
658
659fn append_canonical_json(bytes: &mut Vec<u8>, value: &Value) {
660    match value {
661        Value::Null => bytes.push(0),
662        Value::Bool(value) => {
663            bytes.push(1);
664            bytes.push(u8::from(*value));
665        }
666        Value::Number(value) => {
667            bytes.push(2);
668            append_bytes(bytes, value.to_string().as_bytes());
669        }
670        Value::String(value) => {
671            bytes.push(3);
672            append_bytes(bytes, value.as_bytes());
673        }
674        Value::Array(values) => {
675            bytes.push(4);
676            append_u64(bytes, values.len() as u64);
677            for value in values {
678                append_canonical_json(bytes, value);
679            }
680        }
681        Value::Object(values) => {
682            bytes.push(5);
683            append_u64(bytes, values.len() as u64);
684            let mut entries: Vec<_> = values.iter().collect();
685            entries.sort_unstable_by_key(|(key, _)| *key);
686            for (key, value) in entries {
687                append_bytes(bytes, key.as_bytes());
688                append_canonical_json(bytes, value);
689            }
690        }
691    }
692}
693
694fn evaluation_fingerprint(
695    assertions: &[AssertionFailureKey],
696    judge_fingerprint: Option<&str>,
697) -> String {
698    let mut bytes = Vec::new();
699    append_u64(&mut bytes, assertions.len() as u64);
700    for assertion in assertions {
701        append_bytes(&mut bytes, assertion.assertion.as_bytes());
702        append_bytes(&mut bytes, assertion.expected_fingerprint.as_bytes());
703    }
704    match judge_fingerprint {
705        Some(fingerprint) => {
706            bytes.push(1);
707            append_bytes(&mut bytes, fingerprint.as_bytes());
708        }
709        None => bytes.push(0),
710    }
711    fingerprint("evaluation", &bytes)
712}
713
714fn fingerprint(domain: &str, value: &[u8]) -> String {
715    let mut hasher = Sha256::new();
716    hasher.update(b"rusty.failure-fingerprint.v1\0");
717    hasher.update((domain.len() as u64).to_be_bytes());
718    hasher.update(domain.as_bytes());
719    hasher.update((value.len() as u64).to_be_bytes());
720    hasher.update(value);
721    format_digest("fp_", &hasher.finalize())
722}
723
724fn signature_id(signature: &FailureSignature) -> String {
725    let encoded = serde_json::to_vec(signature).expect("serializing a signature cannot fail");
726    let mut hasher = Sha256::new();
727    hasher.update(b"rusty.failure-cluster.v1\0");
728    hasher.update((encoded.len() as u64).to_be_bytes());
729    hasher.update(encoded);
730    format_digest("fc_", &hasher.finalize())
731}
732
733fn format_digest(prefix: &str, digest: &[u8]) -> String {
734    let mut result = String::from(prefix);
735    for byte in &digest[..16] {
736        write!(&mut result, "{byte:02x}").expect("writing to a String cannot fail");
737    }
738    result
739}
740
741fn append_u64(target: &mut Vec<u8>, value: u64) {
742    target.extend_from_slice(&value.to_be_bytes());
743}
744
745fn append_bytes(target: &mut Vec<u8>, value: &[u8]) {
746    append_u64(target, value.len() as u64);
747    target.extend_from_slice(value);
748}
749
750fn validate_fingerprint(label: &str, fingerprint: &str) -> Result<()> {
751    if fingerprint.len() != 35
752        || !fingerprint.starts_with("fp_")
753        || !fingerprint[3..]
754            .bytes()
755            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
756    {
757        return clustering_error(format!("{label} fingerprint is malformed"));
758    }
759    Ok(())
760}
761
762fn member_key(member: &FailureOccurrence) -> (&str, usize) {
763    (&member.case_id, member.repetition)
764}
765
766fn cluster_sort_key(cluster: &FailureCluster) -> (std::cmp::Reverse<usize>, &FailureSignature) {
767    (std::cmp::Reverse(cluster.occurrences), &cluster.signature)
768}
769
770fn is_sorted_unique<T: Ord>(values: &[T]) -> bool {
771    values.windows(2).all(|pair| pair[0] < pair[1])
772}
773
774fn require_non_empty(label: &str, value: &str) -> Result<()> {
775    if value.trim().is_empty() {
776        return clustering_error(format!("{label} must not be empty"));
777    }
778    Ok(())
779}
780
781fn same_float(actual: f64, expected: f64) -> bool {
782    if !actual.is_finite() || !expected.is_finite() {
783        return false;
784    }
785    if actual == expected {
786        return true;
787    }
788    let scale = actual.abs().max(expected.abs());
789    (actual - expected).abs() <= 16.0 * f64::EPSILON * scale
790}
791
792fn reports_match(actual: &FailureClusterReport, expected: &FailureClusterReport) -> bool {
793    actual.format_version == expected.format_version
794        && actual.experiment == expected.experiment
795        && actual.dataset_name == expected.dataset_name
796        && actual.dataset_version == expected.dataset_version
797        && actual.total_runs == expected.total_runs
798        && actual.total_failures == expected.total_failures
799        && actual.clusters.len() == expected.clusters.len()
800        && actual
801            .clusters
802            .iter()
803            .zip(&expected.clusters)
804            .all(|(actual, expected)| {
805                actual.id == expected.id
806                    && actual.signature == expected.signature
807                    && actual.occurrences == expected.occurrences
808                    && same_float(actual.share_of_failures, expected.share_of_failures)
809                    && actual.members == expected.members
810            })
811}
812
813fn clustering_error<T>(message: impl Into<String>) -> Result<T> {
814    Err(EvalError::Clustering(message.into()))
815}
816
817#[derive(Deserialize)]
818struct FormatHeader {
819    format_version: u64,
820}
821
822#[derive(Deserialize)]
823#[serde(deny_unknown_fields)]
824struct AssertionFailureKeyWire {
825    assertion: String,
826    expected_fingerprint: String,
827}
828
829impl From<AssertionFailureKeyWire> for AssertionFailureKey {
830    fn from(value: AssertionFailureKeyWire) -> Self {
831        Self {
832            assertion: value.assertion,
833            expected_fingerprint: value.expected_fingerprint,
834        }
835    }
836}
837
838#[derive(Deserialize)]
839#[serde(tag = "cause", rename_all = "snake_case", deny_unknown_fields)]
840enum FailureCauseWire {
841    Evaluation {
842        fingerprint: String,
843    },
844    Interrupted,
845    Execution {
846        category: ExecutionFailureCategory,
847        fingerprint: String,
848    },
849}
850
851impl From<FailureCauseWire> for FailureCause {
852    fn from(value: FailureCauseWire) -> Self {
853        match value {
854            FailureCauseWire::Evaluation { fingerprint } => Self::Evaluation { fingerprint },
855            FailureCauseWire::Interrupted => Self::Interrupted,
856            FailureCauseWire::Execution {
857                category,
858                fingerprint,
859            } => Self::Execution {
860                category,
861                fingerprint,
862            },
863        }
864    }
865}
866
867#[derive(Deserialize)]
868#[serde(deny_unknown_fields)]
869struct FailureSignatureWire {
870    termination: FailureTermination,
871    cause: FailureCauseWire,
872    failed_assertions: Vec<AssertionFailureKeyWire>,
873    judge_fingerprint: Option<String>,
874}
875
876impl From<FailureSignatureWire> for FailureSignature {
877    fn from(value: FailureSignatureWire) -> Self {
878        Self {
879            termination: value.termination,
880            cause: value.cause.into(),
881            failed_assertions: value
882                .failed_assertions
883                .into_iter()
884                .map(Into::into)
885                .collect(),
886            judge_fingerprint: value.judge_fingerprint,
887        }
888    }
889}
890
891#[derive(Deserialize)]
892#[serde(tag = "evidence", rename_all = "snake_case", deny_unknown_fields)]
893enum FailureEvidenceRefWire {
894    Execution {
895        category: ExecutionFailureCategory,
896        fingerprint: String,
897    },
898    Interrupted,
899    Assertion {
900        assertion: String,
901        expected_fingerprint: String,
902    },
903    Judge {
904        rationale_fingerprint: String,
905    },
906}
907
908impl From<FailureEvidenceRefWire> for FailureEvidenceRef {
909    fn from(value: FailureEvidenceRefWire) -> Self {
910        match value {
911            FailureEvidenceRefWire::Execution {
912                category,
913                fingerprint,
914            } => Self::Execution {
915                category,
916                fingerprint,
917            },
918            FailureEvidenceRefWire::Interrupted => Self::Interrupted,
919            FailureEvidenceRefWire::Assertion {
920                assertion,
921                expected_fingerprint,
922            } => Self::Assertion {
923                assertion,
924                expected_fingerprint,
925            },
926            FailureEvidenceRefWire::Judge {
927                rationale_fingerprint,
928            } => Self::Judge {
929                rationale_fingerprint,
930            },
931        }
932    }
933}
934
935#[derive(Deserialize)]
936#[serde(deny_unknown_fields)]
937struct FailureOccurrenceWire {
938    case_id: String,
939    repetition: usize,
940    tags: Vec<String>,
941    evidence: Vec<FailureEvidenceRefWire>,
942}
943
944impl From<FailureOccurrenceWire> for FailureOccurrence {
945    fn from(value: FailureOccurrenceWire) -> Self {
946        Self {
947            case_id: value.case_id,
948            repetition: value.repetition,
949            tags: value.tags,
950            evidence: value.evidence.into_iter().map(Into::into).collect(),
951        }
952    }
953}
954
955#[derive(Deserialize)]
956#[serde(deny_unknown_fields)]
957struct FailureClusterWire {
958    id: String,
959    signature: FailureSignatureWire,
960    occurrences: usize,
961    share_of_failures: f64,
962    members: Vec<FailureOccurrenceWire>,
963}
964
965impl From<FailureClusterWire> for FailureCluster {
966    fn from(value: FailureClusterWire) -> Self {
967        Self {
968            id: value.id,
969            signature: value.signature.into(),
970            occurrences: value.occurrences,
971            share_of_failures: value.share_of_failures,
972            members: value.members.into_iter().map(Into::into).collect(),
973        }
974    }
975}
976
977#[derive(Deserialize)]
978#[serde(deny_unknown_fields)]
979struct FailureClusterReportWire {
980    format_version: u64,
981    experiment: String,
982    dataset_name: String,
983    dataset_version: String,
984    total_runs: usize,
985    total_failures: usize,
986    clusters: Vec<FailureClusterWire>,
987}
988
989impl FailureClusterReportWire {
990    fn deserialize_report(text: &str) -> Result<FailureClusterReport> {
991        let wire: Self = serde_json::from_str(text)?;
992        Ok(FailureClusterReport {
993            format_version: wire.format_version,
994            experiment: wire.experiment,
995            dataset_name: wire.dataset_name,
996            dataset_version: wire.dataset_version,
997            total_runs: wire.total_runs,
998            total_failures: wire.total_failures,
999            clusters: wire.clusters.into_iter().map(Into::into).collect(),
1000        })
1001    }
1002}