Skip to main content

gitflow_core/
evaluation.rs

1//! Versioned, provider-independent offline decision evaluation.
2
3use std::collections::{BTreeMap, BTreeSet};
4
5use serde::{Deserialize, Serialize};
6use thiserror::Error;
7
8use crate::decision::{
9    DecisionRequest, DecisionResponse, Question, TypedAnswer, state_contains_obvious_credential,
10};
11
12/// Current fixture, saved-response, and report schema version.
13pub const EVALUATION_SCHEMA_VERSION: u32 = 1;
14/// Maximum number of cases in one evaluation suite.
15pub const MAX_EVALUATION_CASES: usize = 500;
16const MIN_CALIBRATION_CASES: u64 = 30;
17
18/// Content-free validation errors for evaluation artifacts.
19#[derive(Debug, Error)]
20#[non_exhaustive]
21pub enum EvaluationError {
22    /// An artifact violates a schema, bound, or privacy rule.
23    #[error("invalid evaluation artifact: {0}")]
24    Invalid(&'static str),
25    /// The requested question or group is absent.
26    #[error("evaluation question or group not found")]
27    NotFound,
28}
29
30/// A versioned set of labeled requests. It contains no provider credentials.
31#[derive(Debug, Clone, Serialize, Deserialize)]
32#[serde(rename_all = "camelCase", deny_unknown_fields)]
33pub struct FixtureSuite {
34    /// Version of this fixture schema.
35    pub schema_version: u32,
36    /// Stable, caller-owned dataset identifier.
37    pub dataset_id: String,
38    /// Labeled cases in the dataset.
39    pub cases: Vec<FixtureCase>,
40}
41
42/// A labeled request, with language, risk, and provenance for grouped analysis.
43#[derive(Debug, Clone, Serialize, Deserialize)]
44#[serde(rename_all = "camelCase", deny_unknown_fields)]
45pub struct FixtureCase {
46    /// Stable identifier, unique within the suite.
47    pub id: String,
48    /// `english`, `chinese`, `mixed`, or `other`.
49    pub language: String,
50    /// `low`, `medium`, `high`, or `critical`.
51    pub risk: String,
52    /// `synthetic`, `public_issue`, or `reviewed_report`.
53    pub source: String,
54    /// Exact request whose question schema is evaluated.
55    pub request: DecisionRequest,
56    /// Reference answers keyed by question ID.
57    pub expected: BTreeMap<String, ExpectedAnswer>,
58}
59
60/// Independently labeled answer for one fixture question.
61#[derive(Debug, Clone, Serialize, Deserialize)]
62#[serde(tag = "type", rename_all = "lowercase", deny_unknown_fields)]
63pub enum ExpectedAnswer {
64    /// Binary ground truth.
65    Noul {
66        /// Whether the proposition is true.
67        yes: bool,
68    },
69    /// Closed-set ground truth.
70    Choice {
71        /// Expected option key.
72        choice: String,
73    },
74    /// Ordered-scale ground truth, zero based.
75    Score {
76        /// Expected level index.
77        level: u8,
78    },
79}
80
81/// Versioned saved provider results for fully offline evaluation.
82#[derive(Debug, Clone, Serialize, Deserialize)]
83#[serde(rename_all = "camelCase", deny_unknown_fields)]
84pub struct SavedResponses {
85    /// Version of the response schema.
86    pub schema_version: u32,
87    /// Dataset identifier that these results belong to.
88    pub dataset_id: String,
89    /// Provider identifier, independent of the model version.
90    pub provider: String,
91    /// Input token price; omitted when unknown.
92    #[serde(skip_serializing_if = "Option::is_none")]
93    pub input_price_per_million_usd: Option<f64>,
94    /// Output token price; omitted when unknown.
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub output_price_per_million_usd: Option<f64>,
97    /// One outcome per observed case. Missing cases are allowed and reported.
98    pub results: Vec<SavedResult>,
99}
100
101/// One saved case result.
102#[derive(Debug, Clone, Serialize, Deserialize)]
103#[serde(rename_all = "camelCase", deny_unknown_fields)]
104pub struct SavedResult {
105    /// Fixture case identifier.
106    pub case_id: String,
107    /// Valid response or content-free provider failure.
108    pub outcome: SavedOutcome,
109}
110
111/// Explicit success or failure, without raw provider errors or request content.
112#[derive(Debug, Clone, Serialize, Deserialize)]
113#[serde(tag = "status", rename_all = "lowercase", deny_unknown_fields)]
114pub enum SavedOutcome {
115    /// Valid provider response and observed end-to-end latency.
116    Ok {
117        /// Typed response.
118        response: DecisionResponse,
119        /// Elapsed milliseconds.
120        #[serde(rename = "latencyMs")]
121        latency_ms: u64,
122    },
123    /// Unavailable or failed request; excluded from accuracy denominators.
124    Error {
125        /// Safe failure class.
126        kind: FailureKind,
127    },
128}
129
130/// Safe failure categories permitted in saved results.
131#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
132#[serde(rename_all = "snake_case")]
133pub enum FailureKind {
134    /// Provider not configured or not available.
135    Unavailable,
136    /// Request timed out.
137    Timeout,
138    /// Transport or HTTP status failure.
139    Transport,
140    /// Response schema violation.
141    InvalidResponse,
142}
143
144impl FixtureSuite {
145    /// Validate schema, labels, bounds, and obvious sensitive content.
146    ///
147    /// # Errors
148    ///
149    /// Returns [`EvaluationError::Invalid`] for a malformed or unsafe suite.
150    pub fn validate(&self) -> Result<(), EvaluationError> {
151        if self.schema_version != EVALUATION_SCHEMA_VERSION
152            || !valid_id(&self.dataset_id)
153            || self.cases.is_empty()
154            || self.cases.len() > MAX_EVALUATION_CASES
155        {
156            return Err(EvaluationError::Invalid("fixture header is invalid"));
157        }
158        let mut ids = BTreeSet::new();
159        for case in &self.cases {
160            if !valid_id(&case.id) || !ids.insert(case.id.as_str()) {
161                return Err(EvaluationError::Invalid(
162                    "case identifier is invalid or repeated",
163                ));
164            }
165            if !["english", "chinese", "mixed", "other"].contains(&case.language.as_str())
166                || !["low", "medium", "high", "critical"].contains(&case.risk.as_str())
167                || !["synthetic", "public_issue", "reviewed_report"].contains(&case.source.as_str())
168            {
169                return Err(EvaluationError::Invalid("case metadata is invalid"));
170            }
171            case.request
172                .validate()
173                .map_err(|_| EvaluationError::Invalid("case request is invalid"))?;
174            let serialized = serde_json::to_string(&case.request)
175                .map_err(|_| EvaluationError::Invalid("case request is invalid"))?;
176            let value = serde_json::to_value(&case.request)
177                .map_err(|_| EvaluationError::Invalid("case request is invalid"))?;
178            if state_contains_obvious_credential(&value) {
179                return Err(EvaluationError::Invalid(
180                    "fixture contains a credential-like value",
181                ));
182            }
183            let lower = serialized.to_ascii_lowercase();
184            if lower.contains("http://") || lower.contains("https://") {
185                return Err(EvaluationError::Invalid("fixture contains a URL"));
186            }
187            if case.expected.keys().ne(case.request.questions.keys()) {
188                return Err(EvaluationError::Invalid("expected answer set is invalid"));
189            }
190            for (id, question) in &case.request.questions {
191                let expected = case
192                    .expected
193                    .get(id)
194                    .ok_or(EvaluationError::Invalid("expected answer missing"))?;
195                match (question, expected) {
196                    (Question::Noul { .. }, ExpectedAnswer::Noul { .. }) => {}
197                    (Question::Choice { criteria, .. }, ExpectedAnswer::Choice { choice })
198                        if criteria.contains_key(choice) => {}
199                    (Question::Score { criteria, .. }, ExpectedAnswer::Score { level })
200                        if usize::from(*level) < criteria.len() => {}
201                    _ => return Err(EvaluationError::Invalid("expected answer is incompatible")),
202                }
203            }
204        }
205        Ok(())
206    }
207}
208
209impl SavedResponses {
210    /// Validate saved outcomes against a fixture without requiring completeness.
211    ///
212    /// # Errors
213    ///
214    /// Returns [`EvaluationError::Invalid`] for unknown, repeated, or malformed results.
215    pub fn validate_against(&self, fixture: &FixtureSuite) -> Result<(), EvaluationError> {
216        if self.schema_version != EVALUATION_SCHEMA_VERSION
217            || self.dataset_id != fixture.dataset_id
218            || !valid_id(&self.provider)
219            || self.results.len() > fixture.cases.len()
220            || self
221                .input_price_per_million_usd
222                .is_some_and(|value| !value.is_finite() || !(0.0..=100_000.0).contains(&value))
223            || self
224                .output_price_per_million_usd
225                .is_some_and(|value| !value.is_finite() || !(0.0..=100_000.0).contains(&value))
226        {
227            return Err(EvaluationError::Invalid("response header is invalid"));
228        }
229        let cases: BTreeMap<&str, &FixtureCase> = fixture
230            .cases
231            .iter()
232            .map(|case| (case.id.as_str(), case))
233            .collect();
234        let mut ids = BTreeSet::new();
235        for result in &self.results {
236            if !ids.insert(result.case_id.as_str()) {
237                return Err(EvaluationError::Invalid("response case is repeated"));
238            }
239            let case = cases
240                .get(result.case_id.as_str())
241                .ok_or(EvaluationError::Invalid("response case is unknown"))?;
242            if let SavedOutcome::Ok {
243                response,
244                latency_ms,
245            } = &result.outcome
246            {
247                if *latency_ms > 300_000 {
248                    return Err(EvaluationError::Invalid("latency is out of bounds"));
249                }
250                response
251                    .validate_against(&case.request)
252                    .map_err(|_| EvaluationError::Invalid("saved response is invalid"))?;
253            }
254        }
255        Ok(())
256    }
257}
258
259fn valid_id(value: &str) -> bool {
260    !value.is_empty()
261        && value.len() <= 64
262        && value
263            .bytes()
264            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.'))
265}
266
267/// Completeness counts; failures and missing records never count as incorrect predictions.
268#[derive(Debug, Clone, Serialize, Deserialize)]
269#[serde(rename_all = "camelCase", deny_unknown_fields)]
270pub struct Completeness {
271    /// Number of labeled fixture cases.
272    pub total_cases: u64,
273    /// Cases with a valid response.
274    pub completed_cases: u64,
275    /// Cases with a saved provider failure.
276    pub failed_cases: u64,
277    /// Cases without any saved record.
278    pub missing_cases: u64,
279    /// Whether every case has a valid response.
280    pub complete: bool,
281}
282
283/// Accuracy and coverage at one observed confidence cutoff.
284#[derive(Debug, Clone, Serialize, Deserialize)]
285#[serde(rename_all = "camelCase", deny_unknown_fields)]
286pub struct AcceptancePoint {
287    /// Inclusive minimum confidence.
288    pub threshold: f64,
289    /// Number of accepted predictions.
290    pub accepted_count: u64,
291    /// Correct accepted predictions.
292    pub accepted_correct: u64,
293    /// Correct share among accepted predictions.
294    pub accepted_accuracy: f64,
295    /// Accepted share of all labeled cases in the group, including failures.
296    pub coverage: f64,
297    /// Cases not automatically accepted at this cutoff, including failures.
298    pub abstention_count: u64,
299    /// Abstained share of all labeled cases in the group.
300    pub abstention_rate: f64,
301    /// Lower 95% Wilson bound for accepted accuracy.
302    pub wilson_lower: f64,
303    /// Upper 95% Wilson bound for accepted accuracy.
304    pub wilson_upper: f64,
305}
306
307/// Metrics for one question and optional language/risk slice.
308#[derive(Debug, Clone, Serialize, Deserialize)]
309#[serde(rename_all = "camelCase", deny_unknown_fields)]
310pub struct QuestionMetrics {
311    /// Caller-owned question identifier.
312    pub question_id: String,
313    /// Language filter, or `None` for all languages.
314    #[serde(skip_serializing_if = "Option::is_none")]
315    pub language: Option<String>,
316    /// Risk filter, or `None` for all risk levels.
317    #[serde(skip_serializing_if = "Option::is_none")]
318    pub risk: Option<String>,
319    /// Labeled cases containing this question.
320    pub total: u64,
321    /// Cases with a valid response.
322    pub observed: u64,
323    /// Exact matches with reference answers.
324    pub correct: u64,
325    /// Correct share among observed cases; absent when no case was observed.
326    #[serde(skip_serializing_if = "Option::is_none")]
327    pub accuracy: Option<f64>,
328    /// Reference label to selected label counts.
329    pub confusion: BTreeMap<String, BTreeMap<String, u64>>,
330    /// False positives by label.
331    pub false_positives: BTreeMap<String, u64>,
332    /// False negatives by label.
333    pub false_negatives: BTreeMap<String, u64>,
334    /// Coverage and accepted accuracy at each observed cutoff.
335    pub acceptance_curve: Vec<AcceptancePoint>,
336}
337
338impl QuestionMetrics {
339    fn empty(question_id: &str, language: Option<&str>, risk: Option<&str>) -> Self {
340        Self {
341            question_id: question_id.to_string(),
342            language: language.map(str::to_string),
343            risk: risk.map(str::to_string),
344            total: 0,
345            observed: 0,
346            correct: 0,
347            accuracy: None,
348            confusion: BTreeMap::new(),
349            false_positives: BTreeMap::new(),
350            false_negatives: BTreeMap::new(),
351            acceptance_curve: Vec::new(),
352        }
353    }
354}
355
356/// Deterministic offline evaluation report.
357#[derive(Debug, Clone, Serialize, Deserialize)]
358#[serde(rename_all = "camelCase", deny_unknown_fields)]
359pub struct EvaluationReport {
360    /// Report schema version.
361    pub schema_version: u32,
362    /// Fixture schema version.
363    pub fixture_schema_version: u32,
364    /// Saved response schema version.
365    pub response_schema_version: u32,
366    /// Dataset identifier.
367    pub dataset_id: String,
368    /// Provider identifier from saved responses.
369    pub provider: String,
370    /// Actual model versions observed, sorted and deduplicated.
371    pub models: Vec<String>,
372    /// Stable FNV-1a hash of case IDs and typed question schemas.
373    pub question_schema_hash: String,
374    /// Stable FNV-1a hash of all labeled fixture content.
375    pub fixture_content_hash: String,
376    /// Valid, failed, and missing case counts.
377    pub completeness: Completeness,
378    /// Per-question and grouped statistics.
379    pub metrics: Vec<QuestionMetrics>,
380    /// Total input tokens from valid responses.
381    pub total_input_tokens: u64,
382    /// Total output tokens from valid responses.
383    pub total_output_tokens: u64,
384    /// Estimated input-token charge, if its price is known.
385    #[serde(skip_serializing_if = "Option::is_none")]
386    pub total_input_cost_usd: Option<f64>,
387    /// Estimated output-token charge, if its price is known.
388    #[serde(skip_serializing_if = "Option::is_none")]
389    pub total_output_cost_usd: Option<f64>,
390    /// Estimated combined charge, only when both prices are known.
391    #[serde(skip_serializing_if = "Option::is_none")]
392    pub total_cost_usd: Option<f64>,
393    /// Median end-to-end latency of valid responses.
394    #[serde(skip_serializing_if = "Option::is_none")]
395    pub latency_p50_ms: Option<f64>,
396    /// Nearest-rank 95th percentile latency of valid responses.
397    #[serde(skip_serializing_if = "Option::is_none")]
398    pub latency_p95_ms: Option<u64>,
399}
400
401impl EvaluationReport {
402    /// Find metrics for a question and a specific language/risk slice.
403    #[must_use]
404    pub fn metric_for(
405        &self,
406        question: &str,
407        language: Option<&str>,
408        risk: Option<&str>,
409    ) -> Option<&QuestionMetrics> {
410        self.metrics.iter().find(|metric| {
411            metric.question_id == question
412                && metric.language.as_deref() == language
413                && metric.risk.as_deref() == risk
414        })
415    }
416
417    /// Validate a report loaded from a file before calibration or comparison.
418    ///
419    /// # Errors
420    ///
421    /// Returns [`EvaluationError::Invalid`] for malformed metadata or statistics.
422    pub fn validate(&self) -> Result<(), EvaluationError> {
423        if self.schema_version != EVALUATION_SCHEMA_VERSION
424            || self.fixture_schema_version != EVALUATION_SCHEMA_VERSION
425            || self.response_schema_version != EVALUATION_SCHEMA_VERSION
426            || !valid_id(&self.dataset_id)
427            || !valid_id(&self.provider)
428            || !self.question_schema_hash.starts_with("fnv1a64:")
429            || self.question_schema_hash.len() != 24
430            || !self.fixture_content_hash.starts_with("fnv1a64:")
431            || self.fixture_content_hash.len() != 24
432        {
433            return Err(EvaluationError::Invalid("report header is invalid"));
434        }
435        if self
436            .completeness
437            .completed_cases
438            .saturating_add(self.completeness.failed_cases)
439            .saturating_add(self.completeness.missing_cases)
440            != self.completeness.total_cases
441            || self.completeness.complete
442                != (self.completeness.completed_cases == self.completeness.total_cases)
443        {
444            return Err(EvaluationError::Invalid("report completeness is invalid"));
445        }
446        for metric in &self.metrics {
447            if !valid_id(&metric.question_id)
448                || metric.total == 0
449                || metric.observed > metric.total
450                || metric.correct > metric.observed
451                || metric.accuracy.is_some_and(|value| !is_probability(value))
452                || metric.acceptance_curve.iter().any(|point| {
453                    !is_probability(point.threshold)
454                        || !is_probability(point.accepted_accuracy)
455                        || !is_probability(point.coverage)
456                        || !is_probability(point.abstention_rate)
457                        || !is_probability(point.wilson_lower)
458                        || !is_probability(point.wilson_upper)
459                        || point.accepted_count > metric.observed
460                        || point.accepted_correct > point.accepted_count
461                        || point.abstention_count
462                            != metric.total.saturating_sub(point.accepted_count)
463                })
464            {
465                return Err(EvaluationError::Invalid("report metric is invalid"));
466            }
467        }
468        Ok(())
469    }
470}
471
472/// Whether calibration may recommend a threshold from the available evidence.
473#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
474#[serde(rename_all = "snake_case")]
475pub enum CalibrationStatus {
476    /// A threshold satisfies the target at the lower confidence bound.
477    Recommended,
478    /// Too few examples to make an automatic-acceptance recommendation.
479    InsufficientEvidence,
480    /// Enough examples, but no threshold satisfies the target.
481    TargetUnmet,
482    /// Missing or failed case results preclude calibration.
483    Incomplete,
484}
485
486/// Per-question calibration suggestion, never applied to production configuration.
487#[derive(Debug, Clone, Serialize, Deserialize)]
488#[serde(rename_all = "camelCase", deny_unknown_fields)]
489pub struct CalibrationResult {
490    /// Status explaining whether a recommendation is available.
491    pub status: CalibrationStatus,
492    /// Question being calibrated.
493    pub question_id: String,
494    /// Requested lower-bound accuracy target.
495    pub target_accuracy: f64,
496    /// Labeled cases in the selected group.
497    pub sample_size: u64,
498    /// Suggested inclusive cutoff, if supported.
499    #[serde(skip_serializing_if = "Option::is_none")]
500    pub threshold: Option<f64>,
501    /// Accepted cases under the suggested cutoff.
502    pub accepted_count: u64,
503    /// Coverage among all labeled cases.
504    pub coverage: f64,
505    /// Correct share among accepted cases.
506    #[serde(skip_serializing_if = "Option::is_none")]
507    pub accepted_accuracy: Option<f64>,
508    /// Lower Wilson 95% bound, if a threshold was selected.
509    #[serde(skip_serializing_if = "Option::is_none")]
510    pub wilson_lower: Option<f64>,
511    /// Upper Wilson 95% bound, if a threshold was selected.
512    #[serde(skip_serializing_if = "Option::is_none")]
513    pub wilson_upper: Option<f64>,
514}
515
516/// One changed metric in a comparable evaluation pair.
517#[derive(Debug, Clone, Serialize, Deserialize)]
518#[serde(rename_all = "camelCase", deny_unknown_fields)]
519pub struct MetricRegression {
520    /// Question with a regression.
521    pub question_id: String,
522    /// Optional language slice.
523    #[serde(skip_serializing_if = "Option::is_none")]
524    pub language: Option<String>,
525    /// Optional risk slice.
526    #[serde(skip_serializing_if = "Option::is_none")]
527    pub risk: Option<String>,
528    /// Metric name, such as `accuracy` or `false_negatives:3`.
529    pub metric: String,
530    /// Baseline value.
531    pub baseline: f64,
532    /// Candidate value.
533    pub candidate: f64,
534}
535
536/// Read-only comparison result; regressions do not change the process exit code.
537#[derive(Debug, Clone, Serialize, Deserialize)]
538#[serde(rename_all = "camelCase", deny_unknown_fields)]
539pub struct ComparisonReport {
540    /// Whether dataset and question schema are compatible.
541    pub comparable: bool,
542    /// Metadata changes and incompatibility reasons.
543    pub changes: Vec<String>,
544    /// Regressed metrics on comparable reports.
545    pub regressions: Vec<MetricRegression>,
546}
547
548fn is_probability(value: f64) -> bool {
549    value.is_finite() && (0.0..=1.0).contains(&value)
550}
551
552type MetricKey = (String, Option<String>, Option<String>);
553
554#[derive(Debug)]
555struct MetricAccumulator {
556    metric: QuestionMetrics,
557    observations: Vec<(f64, bool)>,
558}
559
560/// Evaluate saved responses without network access or provider credentials.
561///
562/// # Errors
563///
564/// Returns [`EvaluationError::Invalid`] if either artifact is invalid.
565#[allow(
566    clippy::too_many_lines,
567    reason = "Single pass keeps case and question accounting together"
568)]
569pub fn evaluate(
570    fixture: &FixtureSuite,
571    saved: &SavedResponses,
572) -> Result<EvaluationReport, EvaluationError> {
573    fixture.validate()?;
574    saved.validate_against(fixture)?;
575    let by_case: BTreeMap<&str, &SavedOutcome> = saved
576        .results
577        .iter()
578        .map(|result| (result.case_id.as_str(), &result.outcome))
579        .collect();
580    let mut metrics: BTreeMap<MetricKey, MetricAccumulator> = BTreeMap::new();
581    let mut models = BTreeSet::new();
582    let mut latencies = Vec::new();
583    let mut completed = 0_u64;
584    let mut failed = 0_u64;
585    let mut missing = 0_u64;
586    let mut input_tokens = 0_u64;
587    let mut output_tokens = 0_u64;
588
589    for case in &fixture.cases {
590        let outcome = by_case.get(case.id.as_str()).copied();
591        let response = match outcome {
592            Some(SavedOutcome::Ok {
593                response,
594                latency_ms,
595            }) => {
596                completed = completed.saturating_add(1);
597                models.insert(response.model.clone());
598                latencies.push(*latency_ms);
599                input_tokens = input_tokens.saturating_add(response.usage.input_tokens);
600                output_tokens = output_tokens.saturating_add(response.usage.output_tokens);
601                Some(response)
602            }
603            Some(SavedOutcome::Error { .. }) => {
604                failed = failed.saturating_add(1);
605                None
606            }
607            None => {
608                missing = missing.saturating_add(1);
609                None
610            }
611        };
612        for (id, expected) in &case.expected {
613            let observed = response.and_then(|value| value.answers.get(id));
614            let question = case
615                .request
616                .questions
617                .get(id)
618                .ok_or(EvaluationError::Invalid("question missing"))?;
619            let truth = expected_label(expected);
620            let prediction = observed
621                .map(|answer| predicted_label(question, answer))
622                .transpose()?;
623            for (language, risk) in [
624                (None, None),
625                (Some(case.language.as_str()), None),
626                (None, Some(case.risk.as_str())),
627                (Some(case.language.as_str()), Some(case.risk.as_str())),
628            ] {
629                let key = (
630                    id.clone(),
631                    language.map(str::to_string),
632                    risk.map(str::to_string),
633                );
634                let entry = metrics.entry(key).or_insert_with(|| MetricAccumulator {
635                    metric: QuestionMetrics::empty(id, language, risk),
636                    observations: Vec::new(),
637                });
638                entry.metric.total = entry.metric.total.saturating_add(1);
639                if let Some((label, confidence)) = &prediction {
640                    entry.metric.observed = entry.metric.observed.saturating_add(1);
641                    let correct = label == &truth;
642                    if correct {
643                        entry.metric.correct = entry.metric.correct.saturating_add(1);
644                    }
645                    *entry
646                        .metric
647                        .confusion
648                        .entry(truth.clone())
649                        .or_default()
650                        .entry(label.clone())
651                        .or_default() += 1;
652                    entry.observations.push((*confidence, correct));
653                }
654            }
655        }
656    }
657
658    let metrics = metrics.into_values().map(finalize_metric).collect();
659    latencies.sort_unstable();
660    let latency_p50_ms = if latencies.is_empty() {
661        None
662    } else {
663        let middle = latencies.len() / 2;
664        if latencies.len() % 2 == 0 {
665            let left = latencies
666                .get(middle.saturating_sub(1))
667                .copied()
668                .unwrap_or(0);
669            let right = latencies.get(middle).copied().unwrap_or(0);
670            Some(f64::midpoint(to_f64(left), to_f64(right)))
671        } else {
672            latencies.get(middle).copied().map(to_f64)
673        }
674    };
675    let latency_p95_ms = if latencies.is_empty() {
676        None
677    } else {
678        let rank = 95_usize
679            .saturating_mul(latencies.len())
680            .div_ceil(100)
681            .saturating_sub(1);
682        latencies.get(rank).copied()
683    };
684    let total_cases = u64::try_from(fixture.cases.len()).unwrap_or(u64::MAX);
685    let report = EvaluationReport {
686        schema_version: EVALUATION_SCHEMA_VERSION,
687        fixture_schema_version: fixture.schema_version,
688        response_schema_version: saved.schema_version,
689        dataset_id: fixture.dataset_id.clone(),
690        provider: saved.provider.clone(),
691        models: models.into_iter().collect(),
692        question_schema_hash: question_schema_hash(fixture)?,
693        fixture_content_hash: content_hash(fixture)?,
694        completeness: Completeness {
695            total_cases,
696            completed_cases: completed,
697            failed_cases: failed,
698            missing_cases: missing,
699            complete: completed == total_cases,
700        },
701        metrics,
702        total_input_tokens: input_tokens,
703        total_output_tokens: output_tokens,
704        total_input_cost_usd: saved
705            .input_price_per_million_usd
706            .map(|price| price * to_f64(input_tokens) / 1_000_000.0),
707        total_output_cost_usd: saved
708            .output_price_per_million_usd
709            .map(|price| price * to_f64(output_tokens) / 1_000_000.0),
710        total_cost_usd: saved
711            .input_price_per_million_usd
712            .zip(saved.output_price_per_million_usd)
713            .map(|(input_price, output_price)| {
714                (input_price * to_f64(input_tokens) + output_price * to_f64(output_tokens))
715                    / 1_000_000.0
716            }),
717        latency_p50_ms,
718        latency_p95_ms,
719    };
720    report.validate()?;
721    Ok(report)
722}
723
724fn expected_label(answer: &ExpectedAnswer) -> String {
725    match answer {
726        ExpectedAnswer::Noul { yes } => if *yes { "yes" } else { "no" }.to_string(),
727        ExpectedAnswer::Choice { choice } => choice.clone(),
728        ExpectedAnswer::Score { level } => level.to_string(),
729    }
730}
731
732fn predicted_label(
733    question: &Question,
734    answer: &TypedAnswer,
735) -> Result<(String, f64), EvaluationError> {
736    match (question, answer) {
737        (Question::Noul { .. }, TypedAnswer::Noul { noul }) => Ok((
738            if *noul >= 0.5 { "yes" } else { "no" }.to_string(),
739            noul.max(1.0 - noul),
740        )),
741        (
742            Question::Choice { .. },
743            TypedAnswer::Choice {
744                choice, confidence, ..
745            },
746        ) => Ok((choice.clone(), *confidence)),
747        (
748            Question::Score { criteria, .. },
749            TypedAnswer::Score {
750                score, confidence, ..
751            },
752        ) => {
753            let index = (0..criteria.len())
754                .find(|level| *score < f64::from(u32::try_from(*level).unwrap_or(u32::MAX)) + 0.5)
755                .unwrap_or_else(|| criteria.len().saturating_sub(1));
756            Ok((index.to_string(), *confidence))
757        }
758        _ => Err(EvaluationError::Invalid("answer type is incompatible")),
759    }
760}
761
762fn finalize_metric(mut accumulator: MetricAccumulator) -> QuestionMetrics {
763    let metric = &mut accumulator.metric;
764    if metric.observed > 0 {
765        metric.accuracy = Some(to_f64(metric.correct) / to_f64(metric.observed));
766    }
767    let labels: BTreeSet<String> = metric
768        .confusion
769        .iter()
770        .flat_map(|(truth, predictions)| {
771            std::iter::once(truth.clone()).chain(predictions.keys().cloned())
772        })
773        .collect();
774    for label in labels {
775        let true_count = metric
776            .confusion
777            .get(&label)
778            .map_or(0, |row| row.values().sum::<u64>());
779        let predicted_count = metric
780            .confusion
781            .values()
782            .map(|row| row.get(&label).copied().unwrap_or(0))
783            .sum::<u64>();
784        let true_positive = metric
785            .confusion
786            .get(&label)
787            .and_then(|row| row.get(&label))
788            .copied()
789            .unwrap_or(0);
790        metric
791            .false_negatives
792            .insert(label.clone(), true_count.saturating_sub(true_positive));
793        metric
794            .false_positives
795            .insert(label, predicted_count.saturating_sub(true_positive));
796    }
797    accumulator
798        .observations
799        .sort_by(|left, right| right.0.total_cmp(&left.0));
800    let mut accepted = 0_u64;
801    let mut correct = 0_u64;
802    for (index, (confidence, is_correct)) in accumulator.observations.iter().enumerate() {
803        accepted = accepted.saturating_add(1);
804        if *is_correct {
805            correct = correct.saturating_add(1);
806        }
807        let next_confidence = accumulator.observations.get(index + 1).map(|next| next.0);
808        if next_confidence.is_none_or(|next| next.total_cmp(confidence).is_ne()) {
809            let (lower, upper) = wilson_interval(correct, accepted);
810            metric.acceptance_curve.push(AcceptancePoint {
811                threshold: *confidence,
812                accepted_count: accepted,
813                accepted_correct: correct,
814                accepted_accuracy: to_f64(correct) / to_f64(accepted),
815                coverage: to_f64(accepted) / to_f64(metric.total),
816                abstention_count: metric.total.saturating_sub(accepted),
817                abstention_rate: to_f64(metric.total.saturating_sub(accepted))
818                    / to_f64(metric.total),
819                wilson_lower: lower,
820                wilson_upper: upper,
821            });
822        }
823    }
824    accumulator.metric
825}
826
827fn wilson_interval(correct: u64, total: u64) -> (f64, f64) {
828    if total == 0 {
829        return (0.0, 1.0);
830    }
831    let n = to_f64(total);
832    let proportion = to_f64(correct) / n;
833    let z = 1.96_f64;
834    let z_squared = z * z;
835    let denominator = 1.0 + z_squared / n;
836    let center = (proportion + z_squared / (2.0 * n)) / denominator;
837    let margin =
838        z * ((proportion * (1.0 - proportion) + z_squared / (4.0 * n)) / n).sqrt() / denominator;
839    ((center - margin).max(0.0), (center + margin).min(1.0))
840}
841
842fn question_schema_hash(fixture: &FixtureSuite) -> Result<String, EvaluationError> {
843    let mut schemas: Vec<(&str, &BTreeMap<String, Question>)> = fixture
844        .cases
845        .iter()
846        .map(|case| (case.id.as_str(), &case.request.questions))
847        .collect();
848    schemas.sort_by_key(|(id, _)| *id);
849    let bytes = serde_json::to_vec(&schemas)
850        .map_err(|_| EvaluationError::Invalid("question schema cannot be serialized"))?;
851    Ok(fnv1a64(&bytes))
852}
853
854fn content_hash(fixture: &FixtureSuite) -> Result<String, EvaluationError> {
855    let mut cases = fixture.cases.clone();
856    cases.sort_by(|left, right| left.id.cmp(&right.id));
857    let bytes = serde_json::to_vec(&cases)
858        .map_err(|_| EvaluationError::Invalid("fixture cannot be serialized"))?;
859    Ok(fnv1a64(&bytes))
860}
861
862fn fnv1a64(bytes: &[u8]) -> String {
863    let hash = bytes.iter().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
864        (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3)
865    });
866    format!("fnv1a64:{hash:016x}")
867}
868
869#[allow(
870    clippy::cast_precision_loss,
871    reason = "Evaluation counts are bounded to 500 cases; token estimates tolerate rounding"
872)]
873fn to_f64(value: u64) -> f64 {
874    value as f64
875}
876
877/// Select a question-specific threshold only when the 95% lower confidence
878/// bound meets the requested accepted-accuracy target on enough examples.
879///
880/// This returns advice; it never edits runtime configuration.
881///
882/// # Errors
883///
884/// Returns [`EvaluationError`] for an invalid report, target, or group.
885pub fn calibrate(
886    report: &EvaluationReport,
887    question: &str,
888    language: Option<&str>,
889    risk: Option<&str>,
890    target_accuracy: f64,
891) -> Result<CalibrationResult, EvaluationError> {
892    report.validate()?;
893    if !target_accuracy.is_finite() || !(0.0..=1.0).contains(&target_accuracy) {
894        return Err(EvaluationError::Invalid("target accuracy is invalid"));
895    }
896    let metric = report
897        .metric_for(question, language, risk)
898        .ok_or(EvaluationError::NotFound)?;
899    let mut result = CalibrationResult {
900        status: CalibrationStatus::InsufficientEvidence,
901        question_id: question.to_string(),
902        target_accuracy,
903        sample_size: metric.total,
904        threshold: None,
905        accepted_count: 0,
906        coverage: 0.0,
907        accepted_accuracy: None,
908        wilson_lower: None,
909        wilson_upper: None,
910    };
911    if !report.completeness.complete || metric.observed != metric.total {
912        result.status = CalibrationStatus::Incomplete;
913        return Ok(result);
914    }
915    if metric.total < MIN_CALIBRATION_CASES {
916        return Ok(result);
917    }
918    let enough: Vec<&AcceptancePoint> = metric
919        .acceptance_curve
920        .iter()
921        .filter(|point| point.accepted_count >= MIN_CALIBRATION_CASES)
922        .collect();
923    if enough.is_empty() {
924        return Ok(result);
925    }
926    if let Some(point) = enough
927        .into_iter()
928        .filter(|point| point.wilson_lower >= target_accuracy)
929        .max_by_key(|point| point.accepted_count)
930    {
931        result.status = CalibrationStatus::Recommended;
932        result.threshold = Some(point.threshold);
933        result.accepted_count = point.accepted_count;
934        result.coverage = point.coverage;
935        result.accepted_accuracy = Some(point.accepted_accuracy);
936        result.wilson_lower = Some(point.wilson_lower);
937        result.wilson_upper = Some(point.wilson_upper);
938    } else {
939        result.status = CalibrationStatus::TargetUnmet;
940    }
941    Ok(result)
942}
943
944/// Compare two saved reports without turning a regression into a CI gate.
945///
946/// Dataset or question-schema changes make reports incomparable. Model and
947/// provider changes are recorded while comparable metric regressions remain
948/// visible to a human caller.
949///
950/// # Errors
951///
952/// Returns [`EvaluationError::Invalid`] for malformed reports.
953pub fn compare(
954    baseline: &EvaluationReport,
955    candidate: &EvaluationReport,
956) -> Result<ComparisonReport, EvaluationError> {
957    baseline.validate()?;
958    candidate.validate()?;
959    let mut changes = Vec::new();
960    let mut comparable = true;
961    if baseline.dataset_id != candidate.dataset_id {
962        changes.push("dataset changed".to_string());
963        comparable = false;
964    }
965    if baseline.question_schema_hash != candidate.question_schema_hash {
966        changes.push("question schema changed".to_string());
967        comparable = false;
968    }
969    if baseline.fixture_content_hash != candidate.fixture_content_hash {
970        changes.push("fixture content changed".to_string());
971        comparable = false;
972    }
973    if baseline.provider != candidate.provider {
974        changes.push("provider changed".to_string());
975    }
976    if baseline.models != candidate.models {
977        changes.push("model version changed".to_string());
978    }
979    if !baseline.completeness.complete || !candidate.completeness.complete {
980        changes.push("one or both reports are incomplete".to_string());
981        comparable = false;
982    }
983    let mut regressions = Vec::new();
984    if comparable {
985        for old in &baseline.metrics {
986            let Some(new) = candidate.metric_for(
987                &old.question_id,
988                old.language.as_deref(),
989                old.risk.as_deref(),
990            ) else {
991                changes.push("metric group changed".to_string());
992                comparable = false;
993                break;
994            };
995            if old.total != new.total {
996                changes.push("metric sample count changed".to_string());
997                comparable = false;
998                break;
999            }
1000            if let (Some(old_accuracy), Some(new_accuracy)) = (old.accuracy, new.accuracy)
1001                && new_accuracy < old_accuracy
1002            {
1003                regressions.push(MetricRegression {
1004                    question_id: old.question_id.clone(),
1005                    language: old.language.clone(),
1006                    risk: old.risk.clone(),
1007                    metric: "accuracy".to_string(),
1008                    baseline: old_accuracy,
1009                    candidate: new_accuracy,
1010                });
1011            }
1012            for point in &old.acceptance_curve {
1013                let candidate_accepted = new
1014                    .acceptance_curve
1015                    .iter()
1016                    .filter(|candidate_point| candidate_point.threshold >= point.threshold)
1017                    .map(|candidate_point| candidate_point.accepted_count)
1018                    .max()
1019                    .unwrap_or(0);
1020                let candidate_coverage = to_f64(candidate_accepted) / to_f64(new.total);
1021                if candidate_coverage < point.coverage {
1022                    regressions.push(MetricRegression {
1023                        question_id: old.question_id.clone(),
1024                        language: old.language.clone(),
1025                        risk: old.risk.clone(),
1026                        metric: format!("coverage_at_threshold:{}", point.threshold),
1027                        baseline: point.coverage,
1028                        candidate: candidate_coverage,
1029                    });
1030                }
1031            }
1032            for (label, old_count) in &old.false_negatives {
1033                let new_count = new.false_negatives.get(label).copied().unwrap_or(0);
1034                if new_count > *old_count {
1035                    regressions.push(MetricRegression {
1036                        question_id: old.question_id.clone(),
1037                        language: old.language.clone(),
1038                        risk: old.risk.clone(),
1039                        metric: format!("false_negatives:{label}"),
1040                        baseline: to_f64(*old_count),
1041                        candidate: to_f64(new_count),
1042                    });
1043                }
1044            }
1045        }
1046    }
1047    if !comparable {
1048        regressions.clear();
1049    }
1050    Ok(ComparisonReport {
1051        comparable,
1052        changes,
1053        regressions,
1054    })
1055}
1056
1057#[cfg(test)]
1058mod tests {
1059    use serde_json::json;
1060
1061    use super::{CalibrationStatus, FixtureSuite, SavedResponses, calibrate, compare, evaluate};
1062
1063    fn fixture() -> FixtureSuite {
1064        serde_json::from_value(json!({
1065            "schemaVersion": 1,
1066            "datasetId": "synthetic_mixed_v1",
1067            "cases": [{
1068                "id": "case_1", "language": "english", "risk": "high", "source": "synthetic",
1069                "request": {
1070                    "state": {"title": "Existing command fails"},
1071                    "questions": {
1072                        "broken": {"type": "noul", "instructions": "Is the existing behavior broken?"},
1073                        "kind": {"type": "choice", "instructions": "Classify the issue", "criteria": {"bug": "Broken behavior", "feature": "New behavior"}},
1074                        "severity": {"type": "score", "instructions": "Rate impact", "criteria": ["Low impact", "Medium impact", "High impact"]}
1075                    }
1076                },
1077                "expected": {
1078                    "broken": {"type": "noul", "yes": true},
1079                    "kind": {"type": "choice", "choice": "bug"},
1080                    "severity": {"type": "score", "level": 2}
1081                }
1082            }]
1083        })).unwrap()
1084    }
1085
1086    fn responses() -> SavedResponses {
1087        serde_json::from_value(json!({
1088            "schemaVersion": 1, "datasetId": "synthetic_mixed_v1",
1089            "provider": "saved_fixture", "inputPricePerMillionUsd": 0.042, "outputPricePerMillionUsd": 0.3,
1090            "results": [{"caseId": "case_1", "outcome": {
1091                "status": "ok", "latencyMs": 120,
1092                "response": {
1093                    "model": "fixture-model-1",
1094                    "answers": {
1095                        "broken": {"type": "noul", "noul": 0.9},
1096                        "kind": {"type": "choice", "choice": "bug", "confidence": 0.9, "probabilities": {"bug": 0.9, "feature": 0.1}},
1097                        "severity": {"type": "score", "score": 1.8, "confidence": 0.8, "legend": {"0": "Low impact", "1": "Medium impact", "2": "High impact"}, "probabilities": {"0": 0.0, "1": 0.2, "2": 0.8}}
1098                    },
1099                    "usage": {"input_tokens": 100, "output_tokens": 20}
1100                }
1101            }}]
1102        })).unwrap()
1103    }
1104
1105    #[test]
1106    fn test_should_evaluate_mixed_questions_offline() {
1107        let report = evaluate(&fixture(), &responses()).unwrap();
1108        assert_eq!(report.completeness.completed_cases, 1);
1109        assert_eq!(report.models, vec!["fixture-model-1"]);
1110        assert_eq!(report.total_input_tokens, 100);
1111        assert!((report.total_cost_usd.unwrap() - 0.000_010_2).abs() < 1e-12);
1112        for question in ["broken", "kind", "severity"] {
1113            let metric = report.metric_for(question, None, None).unwrap();
1114            assert_eq!(metric.correct, 1);
1115            assert_eq!(metric.observed, 1);
1116        }
1117        assert!(
1118            report
1119                .metric_for("kind", Some("english"), Some("high"))
1120                .is_some()
1121        );
1122    }
1123
1124    #[test]
1125    fn test_should_reject_sensitive_fixture() {
1126        let mut suite = fixture();
1127        suite.cases[0].request.state =
1128            json!({"title": "See https://internal.example.local/private"});
1129        assert!(suite.validate().is_err());
1130        suite.cases[0].request.state = json!({"title": format!("{}{}", "apikey_", "x".repeat(40))});
1131        assert!(suite.validate().is_err());
1132        suite = fixture();
1133        if let crate::decision::Question::Noul { instructions } =
1134            suite.cases[0].request.questions.get_mut("broken").unwrap()
1135        {
1136            *instructions = format!("Check {}{}", "apikey_", "x".repeat(40));
1137        }
1138        assert!(suite.validate().is_err());
1139    }
1140
1141    #[test]
1142    fn test_should_mark_missing_response_incomplete() {
1143        let mut saved = responses();
1144        saved.results.clear();
1145        let report = evaluate(&fixture(), &saved).unwrap();
1146        assert_eq!(report.completeness.missing_cases, 1);
1147        assert!(!report.completeness.complete);
1148        assert_eq!(report.metric_for("kind", None, None).unwrap().observed, 0);
1149    }
1150
1151    #[test]
1152    fn test_should_refuse_calibration_on_small_sample() {
1153        let report = evaluate(&fixture(), &responses()).unwrap();
1154        let result = calibrate(&report, "kind", None, None, 0.8).unwrap();
1155        assert_eq!(result.status, CalibrationStatus::InsufficientEvidence);
1156        assert!(result.threshold.is_none());
1157    }
1158
1159    #[test]
1160    fn test_should_recommend_only_when_wilson_bound_meets_target() {
1161        let mut suite = fixture();
1162        let mut saved = responses();
1163        for index in 2..=40 {
1164            let id = format!("case_{index}");
1165            let mut case = suite.cases[0].clone();
1166            case.id.clone_from(&id);
1167            suite.cases.push(case);
1168            let mut result = saved.results[0].clone();
1169            result.case_id = id;
1170            saved.results.push(result);
1171        }
1172        let report = evaluate(&suite, &saved).unwrap();
1173        let recommended = calibrate(&report, "kind", None, None, 0.90).unwrap();
1174        assert_eq!(recommended.status, CalibrationStatus::Recommended);
1175        assert_eq!(recommended.sample_size, 40);
1176        assert_eq!(recommended.accepted_count, 40);
1177        assert!((recommended.coverage - 1.0).abs() < f64::EPSILON);
1178        assert!(recommended.wilson_lower.unwrap() >= 0.90);
1179        let unmet = calibrate(&report, "kind", None, None, 0.99).unwrap();
1180        assert_eq!(unmet.status, CalibrationStatus::TargetUnmet);
1181    }
1182
1183    #[test]
1184    fn test_should_detect_schema_change_and_metric_regression() {
1185        let baseline = evaluate(&fixture(), &responses()).unwrap();
1186        let mut candidate = baseline.clone();
1187        candidate.question_schema_hash = "fnv1a64:0000000000000000".into();
1188        assert!(!compare(&baseline, &candidate).unwrap().comparable);
1189        candidate.question_schema_hash = baseline.question_schema_hash.clone();
1190        candidate.metrics[0].correct = 0;
1191        candidate.metrics[0].accuracy = Some(0.0);
1192        assert!(
1193            !compare(&baseline, &candidate)
1194                .unwrap()
1195                .regressions
1196                .is_empty()
1197        );
1198    }
1199
1200    #[test]
1201    fn test_should_detect_fixture_content_change_with_same_dataset_id() {
1202        let baseline = evaluate(&fixture(), &responses()).unwrap();
1203        let mut changed = fixture();
1204        changed.cases[0].request.state = json!({"title": "A different public issue"});
1205        let candidate = evaluate(&changed, &responses()).unwrap();
1206        let comparison = compare(&baseline, &candidate).unwrap();
1207        assert!(!comparison.comparable);
1208        assert!(
1209            comparison
1210                .changes
1211                .iter()
1212                .any(|change| change == "fixture content changed")
1213        );
1214    }
1215
1216    #[test]
1217    fn test_should_detect_coverage_regression_at_baseline_threshold() {
1218        let baseline = evaluate(&fixture(), &responses()).unwrap();
1219        let mut changed = responses();
1220        if let super::SavedOutcome::Ok { response, .. } = &mut changed.results[0].outcome
1221            && let crate::decision::TypedAnswer::Choice {
1222                confidence,
1223                probabilities,
1224                ..
1225            } = response.answers.get_mut("kind").unwrap()
1226        {
1227            *confidence = 0.6;
1228            probabilities.insert("bug".into(), 0.6);
1229            probabilities.insert("feature".into(), 0.4);
1230        }
1231        let candidate = evaluate(&fixture(), &changed).unwrap();
1232        let comparison = compare(&baseline, &candidate).unwrap();
1233        assert!(comparison.comparable);
1234        assert!(
1235            comparison
1236                .regressions
1237                .iter()
1238                .any(|regression| regression.question_id == "kind"
1239                    && regression.metric.starts_with("coverage_at_threshold:"))
1240        );
1241    }
1242}