runifold_testkit/
evaluation.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    fmt,
4    future::Future,
5    num::NonZeroUsize,
6    pin::Pin,
7    sync::Arc,
8};
9
10use futures_util::{StreamExt, stream};
11use runifold_core::RunId;
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14use thiserror::Error;
15
16/// Owned asynchronous evaluation operation.
17pub type EvaluationFuture<T> = Pin<Box<dyn Future<Output = T> + Send + 'static>>;
18
19/// Evaluation configuration or execution failure.
20#[derive(Clone, Debug, Error, PartialEq)]
21#[non_exhaustive]
22pub enum EvaluationError {
23    /// A required name or version was empty.
24    #[error("{field} must not be empty")]
25    EmptyField {
26        /// Invalid field name.
27        field: &'static str,
28    },
29    /// A dataset contains no cases.
30    #[error("evaluation dataset must contain at least one case")]
31    EmptyDataset,
32    /// A rule scorer contains no rules.
33    #[error("evaluation rule scorer must contain at least one rule")]
34    EmptyRules,
35    /// A serialized report contradicts its per-case evidence.
36    #[error("evaluation report is inconsistent: {message}")]
37    InconsistentReport {
38        /// Stable inconsistency explanation.
39        message: &'static str,
40    },
41    /// A runner without scorers cannot measure quality.
42    #[error("evaluation runner must contain at least one scorer")]
43    NoScorers,
44    /// A case identifier occurs more than once.
45    #[error("duplicate evaluation case id: {case_id}")]
46    DuplicateCase {
47        /// Duplicated case identifier.
48        case_id: String,
49    },
50    /// A scorer name occurs more than once.
51    #[error("duplicate evaluation scorer name: {scorer}")]
52    DuplicateScorer {
53        /// Duplicated scorer name.
54        scorer: String,
55    },
56    /// A score or threshold is not finite and within zero through one.
57    #[error("{field} must be finite and between 0 and 1, got {value}")]
58    InvalidRatio {
59        /// Invalid field name.
60        field: &'static str,
61        /// Rejected value.
62        value: f64,
63    },
64    /// A duration or monetary metric was negative or non-finite.
65    #[error("{field} must be finite and non-negative, got {value}")]
66    InvalidMetric {
67        /// Invalid field name.
68        field: &'static str,
69        /// Rejected value.
70        value: f64,
71    },
72    /// The evaluated target failed to produce an output.
73    #[error("evaluation target failed: {message}")]
74    Target {
75        /// Safe failure explanation.
76        message: String,
77    },
78    /// A scorer could not evaluate one output.
79    #[error("evaluation scorer {scorer} failed: {message}")]
80    Scorer {
81        /// Stable scorer name.
82        scorer: String,
83        /// Safe failure explanation.
84        message: String,
85    },
86    /// Reports from different dataset identities cannot be compared.
87    #[error(
88        "evaluation dataset mismatch: baseline {baseline_name}@{baseline_version}, candidate {candidate_name}@{candidate_version}"
89    )]
90    DatasetMismatch {
91        /// Baseline dataset name.
92        baseline_name: String,
93        /// Baseline dataset version.
94        baseline_version: String,
95        /// Candidate dataset name.
96        candidate_name: String,
97        /// Candidate dataset version.
98        candidate_version: String,
99    },
100}
101
102/// Stable case identity within a dataset.
103#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
104#[serde(transparent)]
105pub struct EvaluationCaseId(String);
106
107impl EvaluationCaseId {
108    /// Creates a non-empty case identifier.
109    ///
110    /// # Errors
111    ///
112    /// Returns [`EvaluationError::EmptyField`] for an empty identifier.
113    pub fn new(value: impl Into<String>) -> Result<Self, EvaluationError> {
114        let value = value.into();
115        ensure_not_empty("case id", &value)?;
116        Ok(Self(value))
117    }
118
119    /// Returns the identifier text.
120    pub fn as_str(&self) -> &str {
121        &self.0
122    }
123}
124
125impl fmt::Display for EvaluationCaseId {
126    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
127        formatter.write_str(&self.0)
128    }
129}
130
131/// One immutable evaluation input and its optional reference answer.
132#[derive(Clone, Debug, Deserialize, Serialize)]
133pub struct EvaluationCase {
134    id: EvaluationCaseId,
135    input: Value,
136    expected: Option<Value>,
137    tags: BTreeSet<String>,
138}
139
140impl EvaluationCase {
141    /// Creates one case without a reference answer.
142    ///
143    /// # Errors
144    ///
145    /// Returns an error when `id` is empty.
146    pub fn new(id: impl Into<String>, input: Value) -> Result<Self, EvaluationError> {
147        Ok(Self {
148            id: EvaluationCaseId::new(id)?,
149            input,
150            expected: None,
151            tags: BTreeSet::new(),
152        })
153    }
154
155    /// Adds a reference answer for deterministic scorers.
156    #[must_use]
157    pub fn with_expected(mut self, expected: Value) -> Self {
158        self.expected = Some(expected);
159        self
160    }
161
162    /// Adds a low-cardinality dataset tag.
163    ///
164    /// # Errors
165    ///
166    /// Returns an error when the tag is empty.
167    pub fn with_tag(mut self, tag: impl Into<String>) -> Result<Self, EvaluationError> {
168        let tag = tag.into();
169        ensure_not_empty("case tag", &tag)?;
170        self.tags.insert(tag);
171        Ok(self)
172    }
173
174    /// Returns the case identifier.
175    pub const fn id(&self) -> &EvaluationCaseId {
176        &self.id
177    }
178
179    /// Returns the target input.
180    pub const fn input(&self) -> &Value {
181        &self.input
182    }
183
184    /// Returns the optional reference answer.
185    pub const fn expected(&self) -> Option<&Value> {
186        self.expected.as_ref()
187    }
188
189    /// Returns stable case tags.
190    pub const fn tags(&self) -> &BTreeSet<String> {
191        &self.tags
192    }
193}
194
195/// Versioned, duplicate-free evaluation dataset.
196#[derive(Clone, Debug, Deserialize, Serialize)]
197pub struct EvaluationDataset {
198    name: String,
199    version: String,
200    cases: Vec<EvaluationCase>,
201}
202
203impl EvaluationDataset {
204    /// Creates a non-empty versioned dataset.
205    ///
206    /// # Errors
207    ///
208    /// Returns an error for empty fields, no cases, or duplicate case IDs.
209    pub fn new(
210        name: impl Into<String>,
211        version: impl Into<String>,
212        cases: Vec<EvaluationCase>,
213    ) -> Result<Self, EvaluationError> {
214        let name = name.into();
215        let version = version.into();
216        ensure_not_empty("dataset name", &name)?;
217        ensure_not_empty("dataset version", &version)?;
218        if cases.is_empty() {
219            return Err(EvaluationError::EmptyDataset);
220        }
221        let mut ids = BTreeSet::new();
222        for case in &cases {
223            ensure_not_empty("case id", case.id.as_str())?;
224            for tag in &case.tags {
225                ensure_not_empty("case tag", tag)?;
226            }
227            if !ids.insert(case.id.clone()) {
228                return Err(EvaluationError::DuplicateCase {
229                    case_id: case.id.to_string(),
230                });
231            }
232        }
233        Ok(Self {
234            name,
235            version,
236            cases,
237        })
238    }
239
240    /// Returns the dataset name.
241    pub fn name(&self) -> &str {
242        &self.name
243    }
244
245    /// Returns the dataset version.
246    pub fn version(&self) -> &str {
247        &self.version
248    }
249
250    /// Returns cases in stable dataset order.
251    pub fn cases(&self) -> &[EvaluationCase] {
252        &self.cases
253    }
254
255    /// Validates a dataset loaded from an external artifact.
256    ///
257    /// # Errors
258    ///
259    /// Returns the same invariant errors as [`Self::new`].
260    pub fn validate(&self) -> Result<(), EvaluationError> {
261        Self::new(&self.name, &self.version, self.cases.clone()).map(|_| ())
262    }
263}
264
265/// Target output plus optional Run/Trace correlation.
266#[derive(Clone, Debug)]
267pub struct EvaluationOutput {
268    value: Value,
269    run_id: Option<RunId>,
270    metadata: BTreeMap<String, Value>,
271    metrics: Option<EvaluationMetrics>,
272}
273
274impl EvaluationOutput {
275    /// Creates an output without Run correlation.
276    pub fn new(value: Value) -> Self {
277        Self {
278            value,
279            run_id: None,
280            metadata: BTreeMap::new(),
281            metrics: None,
282        }
283    }
284
285    /// Correlates this output with a Runifold Run and its trace.
286    #[must_use]
287    pub const fn with_run_id(mut self, run_id: RunId) -> Self {
288        self.run_id = Some(run_id);
289        self
290    }
291
292    /// Adds scorer-visible metadata.
293    ///
294    /// # Errors
295    ///
296    /// Returns an error when the metadata key is empty.
297    pub fn with_metadata(
298        mut self,
299        key: impl Into<String>,
300        value: Value,
301    ) -> Result<Self, EvaluationError> {
302        let key = key.into();
303        ensure_not_empty("output metadata key", &key)?;
304        self.metadata.insert(key, value);
305        Ok(self)
306    }
307
308    /// Returns the canonical output value.
309    pub const fn value(&self) -> &Value {
310        &self.value
311    }
312
313    /// Returns the correlated Run identifier.
314    pub const fn run_id(&self) -> Option<RunId> {
315        self.run_id
316    }
317
318    /// Returns scorer-visible metadata.
319    pub const fn metadata(&self) -> &BTreeMap<String, Value> {
320        &self.metadata
321    }
322
323    /// Attaches host-measured latency and optional Candidate usage.
324    #[must_use]
325    pub const fn with_metrics(mut self, metrics: EvaluationMetrics) -> Self {
326        self.metrics = Some(metrics);
327        self
328    }
329
330    /// Returns resource evidence associated with this execution.
331    pub const fn metrics(&self) -> Option<&EvaluationMetrics> {
332        self.metrics.as_ref()
333    }
334}
335
336/// Non-sensitive resource evidence for one successful target execution.
337#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
338pub struct EvaluationMetrics {
339    /// Host-observed target duration in milliseconds.
340    pub duration_ms: f64,
341    /// Provider-reported input tokens when available.
342    pub input_tokens: Option<u64>,
343    /// Provider-reported output tokens when available.
344    pub output_tokens: Option<u64>,
345    /// Provider- or application-reported cost in US dollars.
346    pub cost_usd: Option<f64>,
347}
348
349impl EvaluationMetrics {
350    /// Creates metrics with host-observed duration and no Provider usage.
351    ///
352    /// # Errors
353    ///
354    /// Returns an error when duration is negative or non-finite.
355    pub fn new(duration_ms: f64) -> Result<Self, EvaluationError> {
356        ensure_non_negative("evaluation duration milliseconds", duration_ms)?;
357        Ok(Self {
358            duration_ms,
359            input_tokens: None,
360            output_tokens: None,
361            cost_usd: None,
362        })
363    }
364
365    /// Adds Provider token usage.
366    #[must_use]
367    pub const fn with_tokens(mut self, input_tokens: u64, output_tokens: u64) -> Self {
368        self.input_tokens = Some(input_tokens);
369        self.output_tokens = Some(output_tokens);
370        self
371    }
372
373    /// Adds monetary cost.
374    ///
375    /// # Errors
376    ///
377    /// Returns an error when cost is negative or non-finite.
378    pub fn with_cost_usd(mut self, cost_usd: f64) -> Result<Self, EvaluationError> {
379        ensure_non_negative("evaluation cost USD", cost_usd)?;
380        self.cost_usd = Some(cost_usd);
381        Ok(self)
382    }
383
384    fn validate(&self) -> Result<(), EvaluationError> {
385        ensure_non_negative("evaluation duration milliseconds", self.duration_ms)?;
386        if self.input_tokens.is_some() != self.output_tokens.is_some() {
387            return Err(EvaluationError::InconsistentReport {
388                message: "evaluation token metrics must include input and output together",
389            });
390        }
391        if let Some(cost_usd) = self.cost_usd {
392            ensure_non_negative("evaluation cost USD", cost_usd)?;
393        }
394        Ok(())
395    }
396}
397
398/// Asynchronous system-under-evaluation boundary.
399pub trait EvaluationTarget: Send + Sync {
400    /// Executes one owned case.
401    fn execute(
402        &self,
403        case: EvaluationCase,
404    ) -> EvaluationFuture<Result<EvaluationOutput, EvaluationError>>;
405}
406
407impl<F, Fut> EvaluationTarget for F
408where
409    F: Fn(EvaluationCase) -> Fut + Send + Sync,
410    Fut: Future<Output = Result<EvaluationOutput, EvaluationError>> + Send + 'static,
411{
412    fn execute(
413        &self,
414        case: EvaluationCase,
415    ) -> EvaluationFuture<Result<EvaluationOutput, EvaluationError>> {
416        Box::pin(self(case))
417    }
418}
419
420/// Validated score value and optional evaluator rationale.
421#[derive(Clone, Debug)]
422pub struct ScoreValue {
423    value: f64,
424    rationale: Option<String>,
425}
426
427impl ScoreValue {
428    /// Creates a finite score between zero and one.
429    ///
430    /// # Errors
431    ///
432    /// Returns [`EvaluationError::InvalidRatio`] for an invalid value.
433    pub fn new(value: f64) -> Result<Self, EvaluationError> {
434        ensure_ratio("score", value)?;
435        Ok(Self {
436            value,
437            rationale: None,
438        })
439    }
440
441    /// Returns the normalized score.
442    pub const fn value(&self) -> f64 {
443        self.value
444    }
445
446    /// Returns the optional evaluator rationale.
447    pub fn rationale(&self) -> Option<&str> {
448        self.rationale.as_deref()
449    }
450
451    /// Adds an evaluator rationale.
452    #[must_use]
453    pub fn with_rationale(mut self, rationale: impl Into<String>) -> Self {
454        self.rationale = Some(rationale.into());
455        self
456    }
457}
458
459/// Asynchronous scorer boundary.
460pub trait EvaluationScorer: Send + Sync {
461    /// Stable score name.
462    fn name(&self) -> &str;
463
464    /// Per-case passing threshold.
465    fn threshold(&self) -> f64;
466
467    /// Scores one target output.
468    fn score(
469        &self,
470        case: EvaluationCase,
471        output: EvaluationOutput,
472    ) -> EvaluationFuture<Result<ScoreValue, EvaluationError>>;
473}
474
475/// Closure-backed asynchronous scorer.
476pub struct FnScorer<F> {
477    name: String,
478    threshold: f64,
479    scorer: F,
480}
481
482impl<F> FnScorer<F> {
483    /// Creates a scorer with a stable name and per-case threshold.
484    ///
485    /// # Errors
486    ///
487    /// Returns an error for an empty name or invalid threshold.
488    pub fn new(
489        name: impl Into<String>,
490        threshold: f64,
491        scorer: F,
492    ) -> Result<Self, EvaluationError> {
493        let name = name.into();
494        ensure_not_empty("scorer name", &name)?;
495        ensure_ratio("score threshold", threshold)?;
496        Ok(Self {
497            name,
498            threshold,
499            scorer,
500        })
501    }
502}
503
504impl<F, Fut> EvaluationScorer for FnScorer<F>
505where
506    F: Fn(EvaluationCase, EvaluationOutput) -> Fut + Send + Sync,
507    Fut: Future<Output = Result<ScoreValue, EvaluationError>> + Send + 'static,
508{
509    fn name(&self) -> &str {
510        &self.name
511    }
512
513    fn threshold(&self) -> f64 {
514        self.threshold
515    }
516
517    fn score(
518        &self,
519        case: EvaluationCase,
520        output: EvaluationOutput,
521    ) -> EvaluationFuture<Result<ScoreValue, EvaluationError>> {
522        Box::pin((self.scorer)(case, output))
523    }
524}
525
526/// Deterministic JSON equality scorer.
527#[derive(Clone, Copy, Debug, Default)]
528pub struct JsonExactMatchScorer;
529
530impl EvaluationScorer for JsonExactMatchScorer {
531    fn name(&self) -> &'static str {
532        "json_exact_match"
533    }
534
535    fn threshold(&self) -> f64 {
536        1.0
537    }
538
539    fn score(
540        &self,
541        case: EvaluationCase,
542        output: EvaluationOutput,
543    ) -> EvaluationFuture<Result<ScoreValue, EvaluationError>> {
544        Box::pin(async move {
545            let expected = case.expected.ok_or_else(|| EvaluationError::Scorer {
546                scorer: "json_exact_match".into(),
547                message: "case has no reference answer".into(),
548            })?;
549            ScoreValue::new(if expected == output.value { 1.0 } else { 0.0 })
550        })
551    }
552}
553
554/// One persisted per-case score.
555#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
556pub struct EvaluationScore {
557    /// Stable scorer name.
558    pub name: String,
559    /// Normalized score from zero through one.
560    pub value: f64,
561    /// Per-case passing threshold.
562    pub threshold: f64,
563    /// Whether this score meets its threshold.
564    pub passed: bool,
565    /// Optional evaluator explanation.
566    pub rationale: Option<String>,
567}
568
569mod regression;
570mod runner;
571#[cfg(test)]
572mod tests;
573mod validation;
574
575pub use regression::{MetricRegression, RegressionComparison, RegressionPolicy};
576pub use runner::EvaluationRunner;
577use validation::{
578    ensure_close, ensure_non_negative, ensure_not_empty, ensure_ratio, validate_case_metrics,
579};
580
581/// Evaluation failure stage.
582#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
583#[serde(rename_all = "snake_case")]
584#[non_exhaustive]
585pub enum EvaluationFailureStage {
586    /// The target did not produce an output.
587    Target,
588    /// One scorer failed.
589    Scorer,
590}
591
592/// Safe per-case execution or scorer failure.
593#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
594pub struct EvaluationFailure {
595    /// Failure stage.
596    pub stage: EvaluationFailureStage,
597    /// Scorer name for scorer failures.
598    pub scorer: Option<String>,
599    /// Safe operator-facing explanation.
600    pub message: String,
601}
602
603/// Output-free per-case evaluation result.
604#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
605pub struct EvaluationCaseResult {
606    /// Stable case identity.
607    pub case_id: EvaluationCaseId,
608    /// Run/Trace correlation when the target supplied one.
609    pub run_id: Option<RunId>,
610    /// Host latency and optional Provider usage for successful execution.
611    #[serde(default, skip_serializing_if = "Option::is_none")]
612    pub metrics: Option<EvaluationMetrics>,
613    /// Successful scores sorted by scorer name.
614    pub scores: Vec<EvaluationScore>,
615    /// Target and scorer failures.
616    pub failures: Vec<EvaluationFailure>,
617}
618
619/// Aggregate score statistics.
620#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
621pub struct EvaluationScoreSummary {
622    /// Stable scorer name.
623    pub name: String,
624    /// Cases that produced this score.
625    pub scored_cases: usize,
626    /// Total cases in the dataset.
627    pub total_cases: usize,
628    /// Mean over successfully scored cases.
629    pub mean: f64,
630    /// Passing cases divided by all dataset cases.
631    pub pass_rate: f64,
632}
633
634/// Deterministic candidate evaluation report.
635#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
636pub struct EvaluationReport {
637    /// Dataset name.
638    pub dataset_name: String,
639    /// Dataset version.
640    pub dataset_version: String,
641    /// Candidate model, prompt, Agent, or application version.
642    pub candidate_version: String,
643    /// Target executions that produced output.
644    pub execution_success_rate: f64,
645    /// Case results in dataset order.
646    pub cases: Vec<EvaluationCaseResult>,
647    /// Score summaries sorted by scorer name.
648    pub summaries: Vec<EvaluationScoreSummary>,
649}
650
651impl EvaluationReport {
652    /// Serializes this output-free report as stable, pretty JSON.
653    ///
654    /// # Errors
655    ///
656    /// Returns a JSON error only if serialization unexpectedly fails.
657    pub fn to_json_pretty(&self) -> Result<String, serde_json::Error> {
658        serde_json::to_string_pretty(self)
659    }
660
661    /// Compares this candidate with a baseline of the same dataset identity.
662    ///
663    /// # Errors
664    ///
665    /// Returns [`EvaluationError::DatasetMismatch`] when dataset identities
666    /// differ.
667    pub fn compare(
668        &self,
669        baseline: &Self,
670        policy: &RegressionPolicy,
671    ) -> Result<RegressionComparison, EvaluationError> {
672        self.validate()?;
673        baseline.validate()?;
674        policy.validate()?;
675        if self.dataset_name != baseline.dataset_name
676            || self.dataset_version != baseline.dataset_version
677        {
678            return Err(EvaluationError::DatasetMismatch {
679                baseline_name: baseline.dataset_name.clone(),
680                baseline_version: baseline.dataset_version.clone(),
681                candidate_name: self.dataset_name.clone(),
682                candidate_version: self.dataset_version.clone(),
683            });
684        }
685        let metrics = baseline
686            .summaries
687            .iter()
688            .map(|baseline_summary| {
689                let candidate = self
690                    .summaries
691                    .iter()
692                    .find(|summary| summary.name == baseline_summary.name);
693                let candidate_mean = candidate.map_or(0.0, |summary| summary.mean);
694                let candidate_pass_rate = candidate.map_or(0.0, |summary| summary.pass_rate);
695                let mean_delta = candidate_mean - baseline_summary.mean;
696                let pass_rate_delta = candidate_pass_rate - baseline_summary.pass_rate;
697                MetricRegression {
698                    name: baseline_summary.name.clone(),
699                    baseline_mean: baseline_summary.mean,
700                    candidate_mean,
701                    mean_delta,
702                    baseline_pass_rate: baseline_summary.pass_rate,
703                    candidate_pass_rate,
704                    pass_rate_delta,
705                    passed: mean_delta >= -policy.max_mean_drop
706                        && pass_rate_delta >= -policy.max_pass_rate_drop,
707                }
708            })
709            .collect::<Vec<_>>();
710        let execution_success_rate_delta =
711            self.execution_success_rate - baseline.execution_success_rate;
712        let passed = execution_success_rate_delta >= -policy.max_execution_success_drop
713            && metrics.iter().all(|metric| metric.passed);
714        Ok(RegressionComparison {
715            baseline_version: baseline.candidate_version.clone(),
716            candidate_version: self.candidate_version.clone(),
717            execution_success_rate_delta,
718            metrics,
719            passed,
720        })
721    }
722
723    /// Validates a report loaded from an external artifact.
724    ///
725    /// # Errors
726    ///
727    /// Returns an error for empty identities, invalid ratios, or duplicate
728    /// score summaries.
729    pub fn validate(&self) -> Result<(), EvaluationError> {
730        ensure_not_empty("dataset name", &self.dataset_name)?;
731        ensure_not_empty("dataset version", &self.dataset_version)?;
732        ensure_not_empty("candidate version", &self.candidate_version)?;
733        ensure_ratio("execution success rate", self.execution_success_rate)?;
734        if self.cases.is_empty() {
735            return Err(EvaluationError::InconsistentReport {
736                message: "report contains no cases",
737            });
738        }
739        let total_cases = self.cases.iter().fold(0.0, |total, _| total + 1.0);
740        let mut successful_cases = 0.0;
741        let mut case_ids = BTreeSet::new();
742        let mut aggregate = BTreeMap::<&str, (usize, f64, f64, f64)>::new();
743        for case in &self.cases {
744            ensure_not_empty("report case id", case.case_id.as_str())?;
745            if !case_ids.insert(case.case_id.as_str()) {
746                return Err(EvaluationError::InconsistentReport {
747                    message: "report contains duplicate case IDs",
748                });
749            }
750            if !case
751                .failures
752                .iter()
753                .any(|failure| failure.stage == EvaluationFailureStage::Target)
754            {
755                successful_cases += 1.0;
756            }
757            validate_case_metrics(case)?;
758            let mut score_names = BTreeSet::new();
759            for score in &case.scores {
760                ensure_not_empty("report score name", &score.name)?;
761                ensure_ratio("report score", score.value)?;
762                ensure_ratio("report score threshold", score.threshold)?;
763                if score.passed != (score.value >= score.threshold) {
764                    return Err(EvaluationError::InconsistentReport {
765                        message: "stored score decision contradicts its threshold",
766                    });
767                }
768                if !score_names.insert(score.name.as_str()) {
769                    return Err(EvaluationError::InconsistentReport {
770                        message: "one case contains duplicate score names",
771                    });
772                }
773                let entry = aggregate.entry(&score.name).or_default();
774                entry.0 += 1;
775                entry.1 += score.value;
776                entry.2 += 1.0;
777                entry.3 += if score.passed { 1.0 } else { 0.0 };
778            }
779        }
780        ensure_close(
781            self.execution_success_rate,
782            successful_cases / total_cases,
783            "execution success rate contradicts case failures",
784        )?;
785        let mut names = BTreeSet::new();
786        for summary in &self.summaries {
787            ensure_not_empty("score summary name", &summary.name)?;
788            ensure_ratio("score mean", summary.mean)?;
789            ensure_ratio("score pass rate", summary.pass_rate)?;
790            if !names.insert(summary.name.as_str()) {
791                return Err(EvaluationError::DuplicateScorer {
792                    scorer: summary.name.clone(),
793                });
794            }
795            let Some((scored_cases, total, scored_cases_ratio, passed)) =
796                aggregate.get(summary.name.as_str())
797            else {
798                return Err(EvaluationError::InconsistentReport {
799                    message: "score summary has no per-case evidence",
800                });
801            };
802            if summary.scored_cases != *scored_cases || summary.total_cases != self.cases.len() {
803                return Err(EvaluationError::InconsistentReport {
804                    message: "score summary case counts are inconsistent",
805                });
806            }
807            ensure_close(
808                summary.mean,
809                total / scored_cases_ratio,
810                "score summary mean contradicts case scores",
811            )?;
812            ensure_close(
813                summary.pass_rate,
814                passed / total_cases,
815                "score summary pass rate contradicts case scores",
816            )?;
817        }
818        if names.len() != aggregate.len() {
819            return Err(EvaluationError::InconsistentReport {
820                message: "per-case score is missing its summary",
821            });
822        }
823        Ok(())
824    }
825}
runifold_testkit/evaluation.rs

runifold_testkit/
evaluation.rs
