runifold_testkit/
evaluation.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    fmt,
4    future::Future,
5    num::NonZeroUsize,
6    pin::Pin,
7    sync::Arc,
8};
9
10use futures_util::{StreamExt, stream};
11use runifold_core::RunId;
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14use thiserror::Error;
15
16/// Owned asynchronous evaluation operation.
17pub type EvaluationFuture<T> = Pin<Box<dyn Future<Output = T> + Send + 'static>>;
18
19/// Evaluation configuration or execution failure.
20#[derive(Clone, Debug, Error, PartialEq)]
21#[non_exhaustive]
22pub enum EvaluationError {
23    /// A required name or version was empty.
24    #[error("{field} must not be empty")]
25    EmptyField {
26        /// Invalid field name.
27        field: &'static str,
28    },
29    /// A dataset contains no cases.
30    #[error("evaluation dataset must contain at least one case")]
31    EmptyDataset,
32    /// A rule scorer contains no rules.
33    #[error("evaluation rule scorer must contain at least one rule")]
34    EmptyRules,
35    /// A serialized report contradicts its per-case evidence.
36    #[error("evaluation report is inconsistent: {message}")]
37    InconsistentReport {
38        /// Stable inconsistency explanation.
39        message: &'static str,
40    },
41    /// A runner without scorers cannot measure quality.
42    #[error("evaluation runner must contain at least one scorer")]
43    NoScorers,
44    /// A case identifier occurs more than once.
45    #[error("duplicate evaluation case id: {case_id}")]
46    DuplicateCase {
47        /// Duplicated case identifier.
48        case_id: String,
49    },
50    /// A scorer name occurs more than once.
51    #[error("duplicate evaluation scorer name: {scorer}")]
52    DuplicateScorer {
53        /// Duplicated scorer name.
54        scorer: String,
55    },
56    /// A score or threshold is not finite and within zero through one.
57    #[error("{field} must be finite and between 0 and 1, got {value}")]
58    InvalidRatio {
59        /// Invalid field name.
60        field: &'static str,
61        /// Rejected value.
62        value: f64,
63    },
64    /// A duration or monetary metric was negative or non-finite.
65    #[error("{field} must be finite and non-negative, got {value}")]
66    InvalidMetric {
67        /// Invalid field name.
68        field: &'static str,
69        /// Rejected value.
70        value: f64,
71    },
72    /// The evaluated target failed to produce an output.
73    #[error("evaluation target failed: {message}")]
74    Target {
75        /// Safe failure explanation.
76        message: String,
77    },
78    /// A scorer could not evaluate one output.
79    #[error("evaluation scorer {scorer} failed: {message}")]
80    Scorer {
81        /// Stable scorer name.
82        scorer: String,
83        /// Safe failure explanation.
84        message: String,
85    },
86    /// Reports from different dataset identities cannot be compared.
87    #[error(
88        "evaluation dataset mismatch: baseline {baseline_name}@{baseline_version}, candidate {candidate_name}@{candidate_version}"
89    )]
90    DatasetMismatch {
91        /// Baseline dataset name.
92        baseline_name: String,
93        /// Baseline dataset version.
94        baseline_version: String,
95        /// Candidate dataset name.
96        candidate_name: String,
97        /// Candidate dataset version.
98        candidate_version: String,
99    },
100}
101
102/// Stable case identity within a dataset.
103#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
104#[serde(transparent)]
105pub struct EvaluationCaseId(String);
106
107impl EvaluationCaseId {
108    /// Creates a non-empty case identifier.
109    ///
110    /// # Errors
111    ///
112    /// Returns [`EvaluationError::EmptyField`] for an empty identifier.
113    pub fn new(value: impl Into<String>) -> Result<Self, EvaluationError> {
114        let value = value.into();
115        ensure_not_empty("case id", &value)?;
116        Ok(Self(value))
117    }
118
119    /// Returns the identifier text.
120    pub fn as_str(&self) -> &str {
121        &self.0
122    }
123}
124
125impl fmt::Display for EvaluationCaseId {
126    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
127        formatter.write_str(&self.0)
128    }
129}
130
131/// One immutable evaluation input and its optional reference answer.
132#[derive(Clone, Debug, Deserialize, Serialize)]
133pub struct EvaluationCase {
134    id: EvaluationCaseId,
135    input: Value,
136    expected: Option<Value>,
137    tags: BTreeSet<String>,
138}
139
140impl EvaluationCase {
141    /// Creates one case without a reference answer.
142    ///
143    /// # Errors
144    ///
145    /// Returns an error when `id` is empty.
146    pub fn new(id: impl Into<String>, input: Value) -> Result<Self, EvaluationError> {
147        Ok(Self {
148            id: EvaluationCaseId::new(id)?,
149            input,
150            expected: None,
151            tags: BTreeSet::new(),
152        })
153    }
154
155    /// Adds a reference answer for deterministic scorers.
156    #[must_use]
157    pub fn with_expected(mut self, expected: Value) -> Self {
158        self.expected = Some(expected);
159        self
160    }
161
162    /// Adds a low-cardinality dataset tag.
163    ///
164    /// # Errors
165    ///
166    /// Returns an error when the tag is empty.
167    pub fn with_tag(mut self, tag: impl Into<String>) -> Result<Self, EvaluationError> {
168        let tag = tag.into();
169        ensure_not_empty("case tag", &tag)?;
170        self.tags.insert(tag);
171        Ok(self)
172    }
173
174    /// Returns the case identifier.
175    pub const fn id(&self) -> &EvaluationCaseId {
176        &self.id
177    }
178
179    /// Returns the target input.
180    pub const fn input(&self) -> &Value {
181        &self.input
182    }
183
184    /// Returns the optional reference answer.
185    pub const fn expected(&self) -> Option<&Value> {
186        self.expected.as_ref()
187    }
188
189    /// Returns stable case tags.
190    pub const fn tags(&self) -> &BTreeSet<String> {
191        &self.tags
192    }
193}
194
195/// Versioned, duplicate-free evaluation dataset.
196#[derive(Clone, Debug, Deserialize, Serialize)]
197pub struct EvaluationDataset {
198    name: String,
199    version: String,
200    cases: Vec<EvaluationCase>,
201}
202
203impl EvaluationDataset {
204    /// Creates a non-empty versioned dataset.
205    ///
206    /// # Errors
207    ///
208    /// Returns an error for empty fields, no cases, or duplicate case IDs.
209    pub fn new(
210        name: impl Into<String>,
211        version: impl Into<String>,
212        cases: Vec<EvaluationCase>,
213    ) -> Result<Self, EvaluationError> {
214        let name = name.into();
215        let version = version.into();
216        ensure_not_empty("dataset name", &name)?;
217        ensure_not_empty("dataset version", &version)?;
218        if cases.is_empty() {
219            return Err(EvaluationError::EmptyDataset);
220        }
221        let mut ids = BTreeSet::new();
222        for case in &cases {
223            ensure_not_empty("case id", case.id.as_str())?;
224            for tag in &case.tags {
225                ensure_not_empty("case tag", tag)?;
226            }
227            if !ids.insert(case.id.clone()) {
228                return Err(EvaluationError::DuplicateCase {
229                    case_id: case.id.to_string(),
230                });
231            }
232        }
233        Ok(Self {
234            name,
235            version,
236            cases,
237        })
238    }
239
240    /// Returns the dataset name.
241    pub fn name(&self) -> &str {
242        &self.name
243    }
244
245    /// Returns the dataset version.
246    pub fn version(&self) -> &str {
247        &self.version
248    }
249
250    /// Returns cases in stable dataset order.
251    pub fn cases(&self) -> &[EvaluationCase] {
252        &self.cases
253    }
254
255    /// Validates a dataset loaded from an external artifact.
256    ///
257    /// # Errors
258    ///
259    /// Returns the same invariant errors as [`Self::new`].
260    pub fn validate(&self) -> Result<(), EvaluationError> {
261        Self::new(&self.name, &self.version, self.cases.clone()).map(|_| ())
262    }
263}
264
265/// Target output plus optional Run/Trace correlation.
266#[derive(Clone, Debug)]
267pub struct EvaluationOutput {
268    value: Value,
269    run_id: Option<RunId>,
270    metadata: BTreeMap<String, Value>,
271    metrics: Option<EvaluationMetrics>,
272}
273
274impl EvaluationOutput {
275    /// Creates an output without Run correlation.
276    pub fn new(value: Value) -> Self {
277        Self {
278            value,
279            run_id: None,
280            metadata: BTreeMap::new(),
281            metrics: None,
282        }
283    }
284
285    /// Correlates this output with a Runifold Run and its trace.
286    #[must_use]
287    pub const fn with_run_id(mut self, run_id: RunId) -> Self {
288        self.run_id = Some(run_id);
289        self
290    }
291
292    /// Adds scorer-visible metadata.
293    ///
294    /// # Errors
295    ///
296    /// Returns an error when the metadata key is empty.
297    pub fn with_metadata(
298        mut self,
299        key: impl Into<String>,
300        value: Value,
301    ) -> Result<Self, EvaluationError> {
302        let key = key.into();
303        ensure_not_empty("output metadata key", &key)?;
304        self.metadata.insert(key, value);
305        Ok(self)
306    }
307
308    /// Returns the canonical output value.
309    pub const fn value(&self) -> &Value {
310        &self.value
311    }
312
313    /// Returns the correlated Run identifier.
314    pub const fn run_id(&self) -> Option<RunId> {
315        self.run_id
316    }
317
318    /// Returns scorer-visible metadata.
319    pub const fn metadata(&self) -> &BTreeMap<String, Value> {
320        &self.metadata
321    }
322
323    /// Attaches host-measured latency and optional Candidate usage.
324    #[must_use]
325    pub const fn with_metrics(mut self, metrics: EvaluationMetrics) -> Self {
326        self.metrics = Some(metrics);
327        self
328    }
329
330    /// Returns resource evidence associated with this execution.
331    pub const fn metrics(&self) -> Option<&EvaluationMetrics> {
332        self.metrics.as_ref()
333    }
334}
335
336/// Non-sensitive resource evidence for one successful target execution.
337#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
338pub struct EvaluationMetrics {
339    /// Host-observed target duration in milliseconds.
340    pub duration_ms: f64,
341    /// Provider-reported input tokens when available.
342    pub input_tokens: Option<u64>,
343    /// Provider-reported output tokens when available.
344    pub output_tokens: Option<u64>,
345    /// Provider- or application-reported cost in US dollars.
346    pub cost_usd: Option<f64>,
347}
348
349impl EvaluationMetrics {
350    /// Creates metrics with host-observed duration and no Provider usage.
351    ///
352    /// # Errors
353    ///
354    /// Returns an error when duration is negative or non-finite.
355    pub fn new(duration_ms: f64) -> Result<Self, EvaluationError> {
356        ensure_non_negative("evaluation duration milliseconds", duration_ms)?;
357        Ok(Self {
358            duration_ms,
359            input_tokens: None,
360            output_tokens: None,
361            cost_usd: None,
362        })
363    }
364
365    /// Adds Provider token usage.
366    #[must_use]
367    pub const fn with_tokens(mut self, input_tokens: u64, output_tokens: u64) -> Self {
368        self.input_tokens = Some(input_tokens);
369        self.output_tokens = Some(output_tokens);
370        self
371    }
372
373    /// Adds monetary cost.
374    ///
375    /// # Errors
376    ///
377    /// Returns an error when cost is negative or non-finite.
378    pub fn with_cost_usd(mut self, cost_usd: f64) -> Result<Self, EvaluationError> {
379        ensure_non_negative("evaluation cost USD", cost_usd)?;
380        self.cost_usd = Some(cost_usd);
381        Ok(self)
382    }
383
384    fn validate(&self) -> Result<(), EvaluationError> {
385        ensure_non_negative("evaluation duration milliseconds", self.duration_ms)?;
386        if self.input_tokens.is_some() != self.output_tokens.is_some() {
387            return Err(EvaluationError::InconsistentReport {
388                message: "evaluation token metrics must include input and output together",
389            });
390        }
391        if let Some(cost_usd) = self.cost_usd {
392            ensure_non_negative("evaluation cost USD", cost_usd)?;
393        }
394        Ok(())
395    }
396}
397
398/// Asynchronous system-under-evaluation boundary.
399pub trait EvaluationTarget: Send + Sync {
400    /// Executes one owned case.
401    fn execute(
402        &self,
403        case: EvaluationCase,
404    ) -> EvaluationFuture<Result<EvaluationOutput, EvaluationError>>;
405}
406
407impl<F, Fut> EvaluationTarget for F
408where
409    F: Fn(EvaluationCase) -> Fut + Send + Sync,
410    Fut: Future<Output = Result<EvaluationOutput, EvaluationError>> + Send + 'static,
411{
412    fn execute(
413        &self,
414        case: EvaluationCase,
415    ) -> EvaluationFuture<Result<EvaluationOutput, EvaluationError>> {
416        Box::pin(self(case))
417    }
418}
419
420/// Validated score value and optional evaluator rationale.
421#[derive(Clone, Debug)]
422pub struct ScoreValue {
423    value: f64,
424    rationale: Option<String>,
425}
426
427impl ScoreValue {
428    /// Creates a finite score between zero and one.
429    ///
430    /// # Errors
431    ///
432    /// Returns [`EvaluationError::InvalidRatio`] for an invalid value.
433    pub fn new(value: f64) -> Result<Self, EvaluationError> {
434        ensure_ratio("score", value)?;
435        Ok(Self {
436            value,
437            rationale: None,
438        })
439    }
440
441    /// Returns the normalized score.
442    pub const fn value(&self) -> f64 {
443        self.value
444    }
445
446    /// Returns the optional evaluator rationale.
447    pub fn rationale(&self) -> Option<&str> {
448        self.rationale.as_deref()
449    }
450
451    /// Adds an evaluator rationale.
452    #[must_use]
453    pub fn with_rationale(mut self, rationale: impl Into<String>) -> Self {
454        self.rationale = Some(rationale.into());
455        self
456    }
457}
458
459/// Asynchronous scorer boundary.
460pub trait EvaluationScorer: Send + Sync {
461    /// Stable score name.
462    fn name(&self) -> &str;
463
464    /// Per-case passing threshold.
465    fn threshold(&self) -> f64;
466
467    /// Scores one target output.
468    fn score(
469        &self,
470        case: EvaluationCase,
471        output: EvaluationOutput,
472    ) -> EvaluationFuture<Result<ScoreValue, EvaluationError>>;
473}
474
475/// Closure-backed asynchronous scorer.
476pub struct FnScorer<F> {
477    name: String,
478    threshold: f64,
479    scorer: F,
480}
481
482impl<F> FnScorer<F> {
483    /// Creates a scorer with a stable name and per-case threshold.
484    ///
485    /// # Errors
486    ///
487    /// Returns an error for an empty name or invalid threshold.
488    pub fn new(
489        name: impl Into<String>,
490        threshold: f64,
491        scorer: F,
492    ) -> Result<Self, EvaluationError> {
493        let name = name.into();
494        ensure_not_empty("scorer name", &name)?;
495        ensure_ratio("score threshold", threshold)?;
496        Ok(Self {
497            name,
498            threshold,
499            scorer,
500        })
501    }
502}
503
504impl<F, Fut> EvaluationScorer for FnScorer<F>
505where
506    F: Fn(EvaluationCase, EvaluationOutput) -> Fut + Send + Sync,
507    Fut: Future<Output = Result<ScoreValue, EvaluationError>> + Send + 'static,
508{
509    fn name(&self) -> &str {
510        &self.name
511    }
512
513    fn threshold(&self) -> f64 {
514        self.threshold
515    }
516
517    fn score(
518        &self,
519        case: EvaluationCase,
520        output: EvaluationOutput,
521    ) -> EvaluationFuture<Result<ScoreValue, EvaluationError>> {
522        Box::pin((self.scorer)(case, output))
523    }
524}
525
526/// Deterministic JSON equality scorer.
527#[derive(Clone, Copy, Debug, Default)]
528pub struct JsonExactMatchScorer;
529
530impl EvaluationScorer for JsonExactMatchScorer {
531    fn name(&self) -> &'static str {
532        "json_exact_match"
533    }
534
535    fn threshold(&self) -> f64 {
536        1.0
537    }
538
539    fn score(
540        &self,
541        case: EvaluationCase,
542        output: EvaluationOutput,
543    ) -> EvaluationFuture<Result<ScoreValue, EvaluationError>> {
544        Box::pin(async move {
545            let expected = case.expected.ok_or_else(|| EvaluationError::Scorer {
546                scorer: "json_exact_match".into(),
547                message: "case has no reference answer".into(),
548            })?;
549            ScoreValue::new(if expected == output.value { 1.0 } else { 0.0 })
550        })
551    }
552}
553
554/// One persisted per-case score.
555#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
556pub struct EvaluationScore {
557    /// Stable scorer name.
558    pub name: String,
559    /// Normalized score from zero through one.
560    pub value: f64,
561    /// Per-case passing threshold.
562    pub threshold: f64,
563    /// Whether this score meets its threshold.
564    pub passed: bool,
565    /// Optional evaluator explanation.
566    pub rationale: Option<String>,
567}
568
569/// Evaluation failure stage.
570#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
571#[serde(rename_all = "snake_case")]
572#[non_exhaustive]
573pub enum EvaluationFailureStage {
574    /// The target did not produce an output.
575    Target,
576    /// One scorer failed.
577    Scorer,
578}
579
580/// Safe per-case execution or scorer failure.
581#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
582pub struct EvaluationFailure {
583    /// Failure stage.
584    pub stage: EvaluationFailureStage,
585    /// Scorer name for scorer failures.
586    pub scorer: Option<String>,
587    /// Safe operator-facing explanation.
588    pub message: String,
589}
590
591/// Output-free per-case evaluation result.
592#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
593pub struct EvaluationCaseResult {
594    /// Stable case identity.
595    pub case_id: EvaluationCaseId,
596    /// Run/Trace correlation when the target supplied one.
597    pub run_id: Option<RunId>,
598    /// Host latency and optional Provider usage for successful execution.
599    #[serde(default, skip_serializing_if = "Option::is_none")]
600    pub metrics: Option<EvaluationMetrics>,
601    /// Successful scores sorted by scorer name.
602    pub scores: Vec<EvaluationScore>,
603    /// Target and scorer failures.
604    pub failures: Vec<EvaluationFailure>,
605}
606
607/// Aggregate score statistics.
608#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
609pub struct EvaluationScoreSummary {
610    /// Stable scorer name.
611    pub name: String,
612    /// Cases that produced this score.
613    pub scored_cases: usize,
614    /// Total cases in the dataset.
615    pub total_cases: usize,
616    /// Mean over successfully scored cases.
617    pub mean: f64,
618    /// Passing cases divided by all dataset cases.
619    pub pass_rate: f64,
620}
621
622/// Deterministic candidate evaluation report.
623#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
624pub struct EvaluationReport {
625    /// Dataset name.
626    pub dataset_name: String,
627    /// Dataset version.
628    pub dataset_version: String,
629    /// Candidate model, prompt, Agent, or application version.
630    pub candidate_version: String,
631    /// Target executions that produced output.
632    pub execution_success_rate: f64,
633    /// Case results in dataset order.
634    pub cases: Vec<EvaluationCaseResult>,
635    /// Score summaries sorted by scorer name.
636    pub summaries: Vec<EvaluationScoreSummary>,
637}
638
639impl EvaluationReport {
640    /// Serializes this output-free report as stable, pretty JSON.
641    ///
642    /// # Errors
643    ///
644    /// Returns a JSON error only if serialization unexpectedly fails.
645    pub fn to_json_pretty(&self) -> Result<String, serde_json::Error> {
646        serde_json::to_string_pretty(self)
647    }
648
649    /// Compares this candidate with a baseline of the same dataset identity.
650    ///
651    /// # Errors
652    ///
653    /// Returns [`EvaluationError::DatasetMismatch`] when dataset identities
654    /// differ.
655    pub fn compare(
656        &self,
657        baseline: &Self,
658        policy: &RegressionPolicy,
659    ) -> Result<RegressionComparison, EvaluationError> {
660        self.validate()?;
661        baseline.validate()?;
662        policy.validate()?;
663        if self.dataset_name != baseline.dataset_name
664            || self.dataset_version != baseline.dataset_version
665        {
666            return Err(EvaluationError::DatasetMismatch {
667                baseline_name: baseline.dataset_name.clone(),
668                baseline_version: baseline.dataset_version.clone(),
669                candidate_name: self.dataset_name.clone(),
670                candidate_version: self.dataset_version.clone(),
671            });
672        }
673        let metrics = baseline
674            .summaries
675            .iter()
676            .map(|baseline_summary| {
677                let candidate = self
678                    .summaries
679                    .iter()
680                    .find(|summary| summary.name == baseline_summary.name);
681                let candidate_mean = candidate.map_or(0.0, |summary| summary.mean);
682                let candidate_pass_rate = candidate.map_or(0.0, |summary| summary.pass_rate);
683                let mean_delta = candidate_mean - baseline_summary.mean;
684                let pass_rate_delta = candidate_pass_rate - baseline_summary.pass_rate;
685                MetricRegression {
686                    name: baseline_summary.name.clone(),
687                    baseline_mean: baseline_summary.mean,
688                    candidate_mean,
689                    mean_delta,
690                    baseline_pass_rate: baseline_summary.pass_rate,
691                    candidate_pass_rate,
692                    pass_rate_delta,
693                    passed: mean_delta >= -policy.max_mean_drop
694                        && pass_rate_delta >= -policy.max_pass_rate_drop,
695                }
696            })
697            .collect::<Vec<_>>();
698        let execution_success_rate_delta =
699            self.execution_success_rate - baseline.execution_success_rate;
700        let passed = execution_success_rate_delta >= -policy.max_execution_success_drop
701            && metrics.iter().all(|metric| metric.passed);
702        Ok(RegressionComparison {
703            baseline_version: baseline.candidate_version.clone(),
704            candidate_version: self.candidate_version.clone(),
705            execution_success_rate_delta,
706            metrics,
707            passed,
708        })
709    }
710
711    /// Validates a report loaded from an external artifact.
712    ///
713    /// # Errors
714    ///
715    /// Returns an error for empty identities, invalid ratios, or duplicate
716    /// score summaries.
717    pub fn validate(&self) -> Result<(), EvaluationError> {
718        ensure_not_empty("dataset name", &self.dataset_name)?;
719        ensure_not_empty("dataset version", &self.dataset_version)?;
720        ensure_not_empty("candidate version", &self.candidate_version)?;
721        ensure_ratio("execution success rate", self.execution_success_rate)?;
722        if self.cases.is_empty() {
723            return Err(EvaluationError::InconsistentReport {
724                message: "report contains no cases",
725            });
726        }
727        let total_cases = self.cases.iter().fold(0.0, |total, _| total + 1.0);
728        let mut successful_cases = 0.0;
729        let mut case_ids = BTreeSet::new();
730        let mut aggregate = BTreeMap::<&str, (usize, f64, f64, f64)>::new();
731        for case in &self.cases {
732            ensure_not_empty("report case id", case.case_id.as_str())?;
733            if !case_ids.insert(case.case_id.as_str()) {
734                return Err(EvaluationError::InconsistentReport {
735                    message: "report contains duplicate case IDs",
736                });
737            }
738            if !case
739                .failures
740                .iter()
741                .any(|failure| failure.stage == EvaluationFailureStage::Target)
742            {
743                successful_cases += 1.0;
744            }
745            validate_case_metrics(case)?;
746            let mut score_names = BTreeSet::new();
747            for score in &case.scores {
748                ensure_not_empty("report score name", &score.name)?;
749                ensure_ratio("report score", score.value)?;
750                ensure_ratio("report score threshold", score.threshold)?;
751                if score.passed != (score.value >= score.threshold) {
752                    return Err(EvaluationError::InconsistentReport {
753                        message: "stored score decision contradicts its threshold",
754                    });
755                }
756                if !score_names.insert(score.name.as_str()) {
757                    return Err(EvaluationError::InconsistentReport {
758                        message: "one case contains duplicate score names",
759                    });
760                }
761                let entry = aggregate.entry(&score.name).or_default();
762                entry.0 += 1;
763                entry.1 += score.value;
764                entry.2 += 1.0;
765                entry.3 += if score.passed { 1.0 } else { 0.0 };
766            }
767        }
768        ensure_close(
769            self.execution_success_rate,
770            successful_cases / total_cases,
771            "execution success rate contradicts case failures",
772        )?;
773        let mut names = BTreeSet::new();
774        for summary in &self.summaries {
775            ensure_not_empty("score summary name", &summary.name)?;
776            ensure_ratio("score mean", summary.mean)?;
777            ensure_ratio("score pass rate", summary.pass_rate)?;
778            if !names.insert(summary.name.as_str()) {
779                return Err(EvaluationError::DuplicateScorer {
780                    scorer: summary.name.clone(),
781                });
782            }
783            let Some((scored_cases, total, scored_cases_ratio, passed)) =
784                aggregate.get(summary.name.as_str())
785            else {
786                return Err(EvaluationError::InconsistentReport {
787                    message: "score summary has no per-case evidence",
788                });
789            };
790            if summary.scored_cases != *scored_cases || summary.total_cases != self.cases.len() {
791                return Err(EvaluationError::InconsistentReport {
792                    message: "score summary case counts are inconsistent",
793                });
794            }
795            ensure_close(
796                summary.mean,
797                total / scored_cases_ratio,
798                "score summary mean contradicts case scores",
799            )?;
800            ensure_close(
801                summary.pass_rate,
802                passed / total_cases,
803                "score summary pass rate contradicts case scores",
804            )?;
805        }
806        if names.len() != aggregate.len() {
807            return Err(EvaluationError::InconsistentReport {
808                message: "per-case score is missing its summary",
809            });
810        }
811        Ok(())
812    }
813}
814
815/// Allowed relative quality drops.
816#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
817pub struct RegressionPolicy {
818    /// Maximum allowed mean score decrease.
819    pub max_mean_drop: f64,
820    /// Maximum allowed pass-rate decrease.
821    pub max_pass_rate_drop: f64,
822    /// Maximum allowed target execution-success decrease.
823    pub max_execution_success_drop: f64,
824}
825
826impl RegressionPolicy {
827    /// Creates a validated regression policy.
828    ///
829    /// # Errors
830    ///
831    /// Returns an error when any allowed drop is outside zero through one.
832    pub fn new(
833        max_mean_drop: f64,
834        max_pass_rate_drop: f64,
835        max_execution_success_drop: f64,
836    ) -> Result<Self, EvaluationError> {
837        let policy = Self {
838            max_mean_drop,
839            max_pass_rate_drop,
840            max_execution_success_drop,
841        };
842        policy.validate()?;
843        Ok(policy)
844    }
845
846    fn validate(&self) -> Result<(), EvaluationError> {
847        ensure_ratio("maximum mean drop", self.max_mean_drop)?;
848        ensure_ratio("maximum pass-rate drop", self.max_pass_rate_drop)?;
849        ensure_ratio(
850            "maximum execution-success drop",
851            self.max_execution_success_drop,
852        )
853    }
854}
855
856/// One baseline-to-candidate score comparison.
857#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
858pub struct MetricRegression {
859    /// Stable scorer name.
860    pub name: String,
861    /// Baseline mean.
862    pub baseline_mean: f64,
863    /// Candidate mean.
864    pub candidate_mean: f64,
865    /// Candidate minus baseline mean.
866    pub mean_delta: f64,
867    /// Baseline pass rate.
868    pub baseline_pass_rate: f64,
869    /// Candidate pass rate.
870    pub candidate_pass_rate: f64,
871    /// Candidate minus baseline pass rate.
872    pub pass_rate_delta: f64,
873    /// Whether both drops satisfy policy.
874    pub passed: bool,
875}
876
877/// Complete relative regression decision.
878#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
879pub struct RegressionComparison {
880    /// Baseline candidate version.
881    pub baseline_version: String,
882    /// New candidate version.
883    pub candidate_version: String,
884    /// Candidate minus baseline target execution success.
885    pub execution_success_rate_delta: f64,
886    /// Per-score comparisons.
887    pub metrics: Vec<MetricRegression>,
888    /// Whether every relative regression gate passed.
889    pub passed: bool,
890}
891
892/// Concurrent deterministic evaluation orchestrator.
893pub struct EvaluationRunner {
894    target: Arc<dyn EvaluationTarget>,
895    scorers: Vec<Arc<dyn EvaluationScorer>>,
896    concurrency: NonZeroUsize,
897}
898
899impl EvaluationRunner {
900    /// Creates a runner with sequential case execution.
901    pub fn new(target: impl EvaluationTarget + 'static) -> Self {
902        Self {
903            target: Arc::new(target),
904            scorers: Vec::new(),
905            concurrency: NonZeroUsize::MIN,
906        }
907    }
908
909    /// Adds one scorer.
910    #[must_use]
911    pub fn with_scorer(mut self, scorer: impl EvaluationScorer + 'static) -> Self {
912        self.scorers.push(Arc::new(scorer));
913        self
914    }
915
916    /// Bounds concurrently executing cases.
917    #[must_use]
918    pub const fn with_concurrency(mut self, concurrency: NonZeroUsize) -> Self {
919        self.concurrency = concurrency;
920        self
921    }
922
923    /// Evaluates all cases and returns a stable output-free report.
924    ///
925    /// Target and scorer failures are captured per case rather than cancelling
926    /// unrelated cases.
927    ///
928    /// # Errors
929    ///
930    /// Returns an error when `candidate_version` is empty.
931    pub async fn run(
932        &self,
933        dataset: &EvaluationDataset,
934        candidate_version: impl Into<String>,
935    ) -> Result<EvaluationReport, EvaluationError> {
936        let candidate_version = candidate_version.into();
937        ensure_not_empty("candidate version", &candidate_version)?;
938        validate_scorers(&self.scorers)?;
939        let target = Arc::clone(&self.target);
940        let scorers = self.scorers.clone();
941        let mut indexed = stream::iter(dataset.cases.iter().cloned().enumerate())
942            .map(|(index, case)| {
943                let target = Arc::clone(&target);
944                let scorers = scorers.clone();
945                async move { (index, evaluate_case(target, scorers, case).await) }
946            })
947            .buffer_unordered(self.concurrency.get())
948            .collect::<Vec<_>>()
949            .await;
950        indexed.sort_by_key(|(index, _)| *index);
951        let cases = indexed
952            .into_iter()
953            .map(|(_, result)| result)
954            .collect::<Vec<_>>();
955        Ok(build_report(dataset, candidate_version, cases))
956    }
957}
958
959impl fmt::Debug for EvaluationRunner {
960    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
961        formatter
962            .debug_struct("EvaluationRunner")
963            .field("scorers", &self.scorers.len())
964            .field("concurrency", &self.concurrency)
965            .finish_non_exhaustive()
966    }
967}
968
969async fn evaluate_case(
970    target: Arc<dyn EvaluationTarget>,
971    scorers: Vec<Arc<dyn EvaluationScorer>>,
972    case: EvaluationCase,
973) -> EvaluationCaseResult {
974    let output = match target.execute(case.clone()).await {
975        Ok(output) => output,
976        Err(error) => {
977            return EvaluationCaseResult {
978                case_id: case.id,
979                run_id: None,
980                metrics: None,
981                scores: Vec::new(),
982                failures: vec![EvaluationFailure {
983                    stage: EvaluationFailureStage::Target,
984                    scorer: None,
985                    message: error.to_string(),
986                }],
987            };
988        }
989    };
990    let run_id = output.run_id;
991    let metrics = output.metrics.clone();
992    let scorer_concurrency = scorers.len().max(1);
993    let scored = stream::iter(scorers)
994        .map(|scorer| {
995            let case = case.clone();
996            let output = output.clone();
997            async move {
998                let name = scorer.name().to_owned();
999                let threshold = scorer.threshold();
1000                let result = scorer.score(case, output).await;
1001                (name, threshold, result)
1002            }
1003        })
1004        .buffer_unordered(scorer_concurrency)
1005        .collect::<Vec<_>>()
1006        .await;
1007    let mut case_scores = Vec::new();
1008    let mut failures = Vec::new();
1009    for (name, threshold, result) in scored {
1010        match result {
1011            Ok(score) => case_scores.push(EvaluationScore {
1012                name,
1013                value: score.value,
1014                threshold,
1015                passed: score.value >= threshold,
1016                rationale: score.rationale,
1017            }),
1018            Err(error) => failures.push(EvaluationFailure {
1019                stage: EvaluationFailureStage::Scorer,
1020                scorer: Some(name),
1021                message: error.to_string(),
1022            }),
1023        }
1024    }
1025    case_scores.sort_by(|left, right| left.name.cmp(&right.name));
1026    failures.sort_by(|left, right| left.scorer.cmp(&right.scorer));
1027    EvaluationCaseResult {
1028        case_id: case.id,
1029        run_id,
1030        metrics,
1031        scores: case_scores,
1032        failures,
1033    }
1034}
1035
1036fn build_report(
1037    dataset: &EvaluationDataset,
1038    candidate_version: String,
1039    cases: Vec<EvaluationCaseResult>,
1040) -> EvaluationReport {
1041    let total_cases = cases.len();
1042    let total_cases_ratio = cases.iter().fold(0.0, |total, _| total + 1.0);
1043    let successful = cases.iter().fold(0.0, |total, result| {
1044        if result
1045            .failures
1046            .iter()
1047            .any(|failure| failure.stage == EvaluationFailureStage::Target)
1048        {
1049            total
1050        } else {
1051            total + 1.0
1052        }
1053    });
1054    let mut aggregate = BTreeMap::<String, (usize, f64, f64, f64)>::new();
1055    for score in cases.iter().flat_map(|result| &result.scores) {
1056        let entry = aggregate.entry(score.name.clone()).or_default();
1057        entry.0 += 1;
1058        entry.1 += score.value;
1059        entry.2 += 1.0;
1060        entry.3 += if score.passed { 1.0 } else { 0.0 };
1061    }
1062    let summaries = aggregate
1063        .into_iter()
1064        .map(
1065            |(name, (scored_cases, total, scored_cases_ratio, passed))| EvaluationScoreSummary {
1066                name,
1067                scored_cases,
1068                total_cases,
1069                mean: total / scored_cases_ratio,
1070                pass_rate: passed / total_cases_ratio,
1071            },
1072        )
1073        .collect();
1074    EvaluationReport {
1075        dataset_name: dataset.name.clone(),
1076        dataset_version: dataset.version.clone(),
1077        candidate_version,
1078        execution_success_rate: successful / total_cases_ratio,
1079        cases,
1080        summaries,
1081    }
1082}
1083
1084fn ensure_not_empty(field: &'static str, value: &str) -> Result<(), EvaluationError> {
1085    if value.trim().is_empty() {
1086        return Err(EvaluationError::EmptyField { field });
1087    }
1088    Ok(())
1089}
1090
1091fn validate_case_metrics(case: &EvaluationCaseResult) -> Result<(), EvaluationError> {
1092    if case.metrics.is_some()
1093        && case
1094            .failures
1095            .iter()
1096            .any(|failure| failure.stage == EvaluationFailureStage::Target)
1097    {
1098        return Err(EvaluationError::InconsistentReport {
1099            message: "target failure cannot contain successful execution metrics",
1100        });
1101    }
1102    if let Some(metrics) = &case.metrics {
1103        metrics.validate()?;
1104    }
1105    Ok(())
1106}
1107
1108fn ensure_ratio(field: &'static str, value: f64) -> Result<(), EvaluationError> {
1109    if !value.is_finite() || !(0.0..=1.0).contains(&value) {
1110        return Err(EvaluationError::InvalidRatio { field, value });
1111    }
1112    Ok(())
1113}
1114
1115fn ensure_non_negative(field: &'static str, value: f64) -> Result<(), EvaluationError> {
1116    if !value.is_finite() || value < 0.0 {
1117        return Err(EvaluationError::InvalidMetric { field, value });
1118    }
1119    Ok(())
1120}
1121
1122fn ensure_close(actual: f64, expected: f64, message: &'static str) -> Result<(), EvaluationError> {
1123    const REPORT_RATIO_TOLERANCE: f64 = 1e-12;
1124    if (actual - expected).abs() > REPORT_RATIO_TOLERANCE {
1125        return Err(EvaluationError::InconsistentReport { message });
1126    }
1127    Ok(())
1128}
1129
1130fn validate_scorers(scorers: &[Arc<dyn EvaluationScorer>]) -> Result<(), EvaluationError> {
1131    if scorers.is_empty() {
1132        return Err(EvaluationError::NoScorers);
1133    }
1134    let mut names = BTreeSet::new();
1135    for scorer in scorers {
1136        ensure_not_empty("scorer name", scorer.name())?;
1137        ensure_ratio("score threshold", scorer.threshold())?;
1138        if !names.insert(scorer.name()) {
1139            return Err(EvaluationError::DuplicateScorer {
1140                scorer: scorer.name().to_owned(),
1141            });
1142        }
1143    }
1144    Ok(())
1145}
1146
1147#[cfg(test)]
1148mod tests {
1149    use std::num::NonZeroUsize;
1150
1151    use runifold_core::RunId;
1152
1153    use super::{
1154        EvaluationCase, EvaluationDataset, EvaluationError, EvaluationOutput, EvaluationRunner,
1155        JsonExactMatchScorer, RegressionPolicy, ScoreValue,
1156    };
1157
1158    #[test]
1159    fn dataset_rejects_duplicate_case_ids() {
1160        let first = EvaluationCase::new("same", serde_json::json!("one")).unwrap();
1161        let second = EvaluationCase::new("same", serde_json::json!("two")).unwrap();
1162
1163        let error = EvaluationDataset::new("dataset", "1", vec![first, second]).unwrap_err();
1164
1165        assert!(matches!(error, EvaluationError::DuplicateCase { .. }));
1166    }
1167
1168    #[test]
1169    fn score_rejects_non_finite_or_out_of_range_values() {
1170        for value in [-0.1, 1.1, f64::NAN, f64::INFINITY] {
1171            assert!(matches!(
1172                ScoreValue::new(value),
1173                Err(EvaluationError::InvalidRatio { .. })
1174            ));
1175        }
1176    }
1177
1178    #[test]
1179    fn metrics_reject_negative_or_non_finite_values() {
1180        for value in [-0.1, f64::NAN, f64::INFINITY] {
1181            assert!(matches!(
1182                super::EvaluationMetrics::new(value),
1183                Err(EvaluationError::InvalidMetric { .. })
1184            ));
1185        }
1186        assert!(
1187            super::EvaluationMetrics::new(1.0)
1188                .unwrap()
1189                .with_cost_usd(-0.1)
1190                .is_err()
1191        );
1192    }
1193
1194    #[test]
1195    fn runner_requires_at_least_one_scorer() {
1196        let dataset = EvaluationDataset::new(
1197            "answers",
1198            "1",
1199            vec![EvaluationCase::new("one", serde_json::json!("answer")).unwrap()],
1200        )
1201        .unwrap();
1202        let runner = EvaluationRunner::new(|case: EvaluationCase| async move {
1203            Ok(EvaluationOutput::new(case.input().clone()))
1204        });
1205
1206        let error = futures_executor::block_on(runner.run(&dataset, "candidate")).unwrap_err();
1207
1208        assert_eq!(error, EvaluationError::NoScorers);
1209    }
1210
1211    #[test]
1212    fn concurrent_runner_is_ordered_correlated_and_output_free() {
1213        let dataset = EvaluationDataset::new(
1214            "answers",
1215            "2026-07-26",
1216            vec![
1217                EvaluationCase::new("first", serde_json::json!("secret-one"))
1218                    .unwrap()
1219                    .with_expected(serde_json::json!("secret-one")),
1220                EvaluationCase::new("second", serde_json::json!("secret-two"))
1221                    .unwrap()
1222                    .with_expected(serde_json::json!("secret-two")),
1223            ],
1224        )
1225        .unwrap();
1226        let runner = EvaluationRunner::new(|case: EvaluationCase| async move {
1227            let output = EvaluationOutput::new(case.input().clone());
1228            Ok(if case.id().as_str() == "first" {
1229                output.with_run_id(RunId::new())
1230            } else {
1231                output
1232            })
1233        })
1234        .with_scorer(JsonExactMatchScorer)
1235        .with_concurrency(NonZeroUsize::new(2).unwrap());
1236
1237        let report = futures_executor::block_on(runner.run(&dataset, "candidate-a")).unwrap();
1238        let json = report.to_json_pretty().unwrap();
1239
1240        assert_eq!(report.cases[0].case_id.as_str(), "first");
1241        assert_eq!(report.cases[1].case_id.as_str(), "second");
1242        assert!(report.cases[0].run_id.is_some());
1243        assert!(report.cases[1].run_id.is_none());
1244        assert!((report.execution_success_rate - 1.0).abs() < 1e-12);
1245        assert!((report.summaries[0].mean - 1.0).abs() < 1e-12);
1246        assert!(!json.contains("secret-one"));
1247        assert!(!json.contains("secret-two"));
1248    }
1249
1250    #[test]
1251    fn relative_gate_detects_mean_and_pass_rate_regression() {
1252        let baseline = report("baseline", 1.0, 1.0);
1253        let candidate = report("candidate", 0.8, 0.5);
1254        let policy = RegressionPolicy::new(0.05, 0.1, 0.0).unwrap();
1255
1256        let comparison = candidate.compare(&baseline, &policy).unwrap();
1257
1258        assert!(!comparison.passed);
1259        assert!((comparison.metrics[0].mean_delta - -0.2).abs() < 1e-12);
1260        assert!((comparison.metrics[0].pass_rate_delta - -0.5).abs() < 1e-12);
1261    }
1262
1263    #[test]
1264    fn externally_loaded_report_cannot_forge_aggregate_quality() {
1265        let mut forged = report("candidate", 0.8, 0.5);
1266        forged.summaries[0].mean = 1.0;
1267
1268        assert!(matches!(
1269            forged.validate(),
1270            Err(EvaluationError::InconsistentReport { .. })
1271        ));
1272    }
1273
1274    fn report(candidate: &str, mean: f64, pass_rate: f64) -> super::EvaluationReport {
1275        let values = if pass_rate > 0.75 {
1276            [mean, mean]
1277        } else {
1278            [mean - 0.1, mean + 0.1]
1279        };
1280        let cases = values
1281            .into_iter()
1282            .enumerate()
1283            .map(|(index, value)| super::EvaluationCaseResult {
1284                case_id: super::EvaluationCaseId::new(format!("case-{index}")).unwrap(),
1285                run_id: None,
1286                metrics: None,
1287                scores: vec![super::EvaluationScore {
1288                    name: "quality".into(),
1289                    value,
1290                    threshold: 0.8,
1291                    passed: value >= 0.8,
1292                    rationale: None,
1293                }],
1294                failures: Vec::new(),
1295            })
1296            .collect();
1297        super::EvaluationReport {
1298            dataset_name: "answers".into(),
1299            dataset_version: "1".into(),
1300            candidate_version: candidate.into(),
1301            execution_success_rate: 1.0,
1302            cases,
1303            summaries: vec![super::EvaluationScoreSummary {
1304                name: "quality".into(),
1305                scored_cases: 2,
1306                total_cases: 2,
1307                mean,
1308                pass_rate,
1309            }],
1310        }
1311    }
1312}
runifold_testkit/evaluation.rs

runifold_testkit/
evaluation.rs
