1use std::collections::{BTreeMap, BTreeSet};
4
5use serde::{Deserialize, Serialize};
6use thiserror::Error;
7
8use crate::decision::{
9 DecisionRequest, DecisionResponse, Question, TypedAnswer, state_contains_obvious_credential,
10};
11
12pub const EVALUATION_SCHEMA_VERSION: u32 = 1;
14pub const MAX_EVALUATION_CASES: usize = 500;
16const MIN_CALIBRATION_CASES: u64 = 30;
17
18#[derive(Debug, Error)]
20#[non_exhaustive]
21pub enum EvaluationError {
22 #[error("invalid evaluation artifact: {0}")]
24 Invalid(&'static str),
25 #[error("evaluation question or group not found")]
27 NotFound,
28}
29
30#[derive(Debug, Clone, Serialize, Deserialize)]
32#[serde(rename_all = "camelCase", deny_unknown_fields)]
33pub struct FixtureSuite {
34 pub schema_version: u32,
36 pub dataset_id: String,
38 pub cases: Vec<FixtureCase>,
40}
41
42#[derive(Debug, Clone, Serialize, Deserialize)]
44#[serde(rename_all = "camelCase", deny_unknown_fields)]
45pub struct FixtureCase {
46 pub id: String,
48 pub language: String,
50 pub risk: String,
52 pub source: String,
54 pub request: DecisionRequest,
56 pub expected: BTreeMap<String, ExpectedAnswer>,
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
62#[serde(tag = "type", rename_all = "lowercase", deny_unknown_fields)]
63pub enum ExpectedAnswer {
64 Noul {
66 yes: bool,
68 },
69 Choice {
71 choice: String,
73 },
74 Score {
76 level: u8,
78 },
79}
80
81#[derive(Debug, Clone, Serialize, Deserialize)]
83#[serde(rename_all = "camelCase", deny_unknown_fields)]
84pub struct SavedResponses {
85 pub schema_version: u32,
87 pub dataset_id: String,
89 pub provider: String,
91 #[serde(skip_serializing_if = "Option::is_none")]
93 pub input_price_per_million_usd: Option<f64>,
94 #[serde(skip_serializing_if = "Option::is_none")]
96 pub output_price_per_million_usd: Option<f64>,
97 pub results: Vec<SavedResult>,
99}
100
101#[derive(Debug, Clone, Serialize, Deserialize)]
103#[serde(rename_all = "camelCase", deny_unknown_fields)]
104pub struct SavedResult {
105 pub case_id: String,
107 pub outcome: SavedOutcome,
109}
110
111#[derive(Debug, Clone, Serialize, Deserialize)]
113#[serde(tag = "status", rename_all = "lowercase", deny_unknown_fields)]
114pub enum SavedOutcome {
115 Ok {
117 response: DecisionResponse,
119 #[serde(rename = "latencyMs")]
121 latency_ms: u64,
122 },
123 Error {
125 kind: FailureKind,
127 },
128}
129
130#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
132#[serde(rename_all = "snake_case")]
133pub enum FailureKind {
134 Unavailable,
136 Timeout,
138 Transport,
140 InvalidResponse,
142}
143
144impl FixtureSuite {
145 pub fn validate(&self) -> Result<(), EvaluationError> {
151 if self.schema_version != EVALUATION_SCHEMA_VERSION
152 || !valid_id(&self.dataset_id)
153 || self.cases.is_empty()
154 || self.cases.len() > MAX_EVALUATION_CASES
155 {
156 return Err(EvaluationError::Invalid("fixture header is invalid"));
157 }
158 let mut ids = BTreeSet::new();
159 for case in &self.cases {
160 if !valid_id(&case.id) || !ids.insert(case.id.as_str()) {
161 return Err(EvaluationError::Invalid(
162 "case identifier is invalid or repeated",
163 ));
164 }
165 if !["english", "chinese", "mixed", "other"].contains(&case.language.as_str())
166 || !["low", "medium", "high", "critical"].contains(&case.risk.as_str())
167 || !["synthetic", "public_issue", "reviewed_report"].contains(&case.source.as_str())
168 {
169 return Err(EvaluationError::Invalid("case metadata is invalid"));
170 }
171 case.request
172 .validate()
173 .map_err(|_| EvaluationError::Invalid("case request is invalid"))?;
174 let serialized = serde_json::to_string(&case.request)
175 .map_err(|_| EvaluationError::Invalid("case request is invalid"))?;
176 let value = serde_json::to_value(&case.request)
177 .map_err(|_| EvaluationError::Invalid("case request is invalid"))?;
178 if state_contains_obvious_credential(&value) {
179 return Err(EvaluationError::Invalid(
180 "fixture contains a credential-like value",
181 ));
182 }
183 let lower = serialized.to_ascii_lowercase();
184 if lower.contains("http://") || lower.contains("https://") {
185 return Err(EvaluationError::Invalid("fixture contains a URL"));
186 }
187 if case.expected.keys().ne(case.request.questions.keys()) {
188 return Err(EvaluationError::Invalid("expected answer set is invalid"));
189 }
190 for (id, question) in &case.request.questions {
191 let expected = case
192 .expected
193 .get(id)
194 .ok_or(EvaluationError::Invalid("expected answer missing"))?;
195 match (question, expected) {
196 (Question::Noul { .. }, ExpectedAnswer::Noul { .. }) => {}
197 (Question::Choice { criteria, .. }, ExpectedAnswer::Choice { choice })
198 if criteria.contains_key(choice) => {}
199 (Question::Score { criteria, .. }, ExpectedAnswer::Score { level })
200 if usize::from(*level) < criteria.len() => {}
201 _ => return Err(EvaluationError::Invalid("expected answer is incompatible")),
202 }
203 }
204 }
205 Ok(())
206 }
207}
208
209impl SavedResponses {
210 pub fn validate_against(&self, fixture: &FixtureSuite) -> Result<(), EvaluationError> {
216 if self.schema_version != EVALUATION_SCHEMA_VERSION
217 || self.dataset_id != fixture.dataset_id
218 || !valid_id(&self.provider)
219 || self.results.len() > fixture.cases.len()
220 || self
221 .input_price_per_million_usd
222 .is_some_and(|value| !value.is_finite() || !(0.0..=100_000.0).contains(&value))
223 || self
224 .output_price_per_million_usd
225 .is_some_and(|value| !value.is_finite() || !(0.0..=100_000.0).contains(&value))
226 {
227 return Err(EvaluationError::Invalid("response header is invalid"));
228 }
229 let cases: BTreeMap<&str, &FixtureCase> = fixture
230 .cases
231 .iter()
232 .map(|case| (case.id.as_str(), case))
233 .collect();
234 let mut ids = BTreeSet::new();
235 for result in &self.results {
236 if !ids.insert(result.case_id.as_str()) {
237 return Err(EvaluationError::Invalid("response case is repeated"));
238 }
239 let case = cases
240 .get(result.case_id.as_str())
241 .ok_or(EvaluationError::Invalid("response case is unknown"))?;
242 if let SavedOutcome::Ok {
243 response,
244 latency_ms,
245 } = &result.outcome
246 {
247 if *latency_ms > 300_000 {
248 return Err(EvaluationError::Invalid("latency is out of bounds"));
249 }
250 response
251 .validate_against(&case.request)
252 .map_err(|_| EvaluationError::Invalid("saved response is invalid"))?;
253 }
254 }
255 Ok(())
256 }
257}
258
259fn valid_id(value: &str) -> bool {
260 !value.is_empty()
261 && value.len() <= 64
262 && value
263 .bytes()
264 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.'))
265}
266
267#[derive(Debug, Clone, Serialize, Deserialize)]
269#[serde(rename_all = "camelCase", deny_unknown_fields)]
270pub struct Completeness {
271 pub total_cases: u64,
273 pub completed_cases: u64,
275 pub failed_cases: u64,
277 pub missing_cases: u64,
279 pub complete: bool,
281}
282
283#[derive(Debug, Clone, Serialize, Deserialize)]
285#[serde(rename_all = "camelCase", deny_unknown_fields)]
286pub struct AcceptancePoint {
287 pub threshold: f64,
289 pub accepted_count: u64,
291 pub accepted_correct: u64,
293 pub accepted_accuracy: f64,
295 pub coverage: f64,
297 pub abstention_count: u64,
299 pub abstention_rate: f64,
301 pub wilson_lower: f64,
303 pub wilson_upper: f64,
305}
306
307#[derive(Debug, Clone, Serialize, Deserialize)]
309#[serde(rename_all = "camelCase", deny_unknown_fields)]
310pub struct QuestionMetrics {
311 pub question_id: String,
313 #[serde(skip_serializing_if = "Option::is_none")]
315 pub language: Option<String>,
316 #[serde(skip_serializing_if = "Option::is_none")]
318 pub risk: Option<String>,
319 pub total: u64,
321 pub observed: u64,
323 pub correct: u64,
325 #[serde(skip_serializing_if = "Option::is_none")]
327 pub accuracy: Option<f64>,
328 pub confusion: BTreeMap<String, BTreeMap<String, u64>>,
330 pub false_positives: BTreeMap<String, u64>,
332 pub false_negatives: BTreeMap<String, u64>,
334 pub acceptance_curve: Vec<AcceptancePoint>,
336}
337
338impl QuestionMetrics {
339 fn empty(question_id: &str, language: Option<&str>, risk: Option<&str>) -> Self {
340 Self {
341 question_id: question_id.to_string(),
342 language: language.map(str::to_string),
343 risk: risk.map(str::to_string),
344 total: 0,
345 observed: 0,
346 correct: 0,
347 accuracy: None,
348 confusion: BTreeMap::new(),
349 false_positives: BTreeMap::new(),
350 false_negatives: BTreeMap::new(),
351 acceptance_curve: Vec::new(),
352 }
353 }
354}
355
356#[derive(Debug, Clone, Serialize, Deserialize)]
358#[serde(rename_all = "camelCase", deny_unknown_fields)]
359pub struct EvaluationReport {
360 pub schema_version: u32,
362 pub fixture_schema_version: u32,
364 pub response_schema_version: u32,
366 pub dataset_id: String,
368 pub provider: String,
370 pub models: Vec<String>,
372 pub question_schema_hash: String,
374 pub fixture_content_hash: String,
376 pub completeness: Completeness,
378 pub metrics: Vec<QuestionMetrics>,
380 pub total_input_tokens: u64,
382 pub total_output_tokens: u64,
384 #[serde(skip_serializing_if = "Option::is_none")]
386 pub total_input_cost_usd: Option<f64>,
387 #[serde(skip_serializing_if = "Option::is_none")]
389 pub total_output_cost_usd: Option<f64>,
390 #[serde(skip_serializing_if = "Option::is_none")]
392 pub total_cost_usd: Option<f64>,
393 #[serde(skip_serializing_if = "Option::is_none")]
395 pub latency_p50_ms: Option<f64>,
396 #[serde(skip_serializing_if = "Option::is_none")]
398 pub latency_p95_ms: Option<u64>,
399}
400
401impl EvaluationReport {
402 #[must_use]
404 pub fn metric_for(
405 &self,
406 question: &str,
407 language: Option<&str>,
408 risk: Option<&str>,
409 ) -> Option<&QuestionMetrics> {
410 self.metrics.iter().find(|metric| {
411 metric.question_id == question
412 && metric.language.as_deref() == language
413 && metric.risk.as_deref() == risk
414 })
415 }
416
417 pub fn validate(&self) -> Result<(), EvaluationError> {
423 if self.schema_version != EVALUATION_SCHEMA_VERSION
424 || self.fixture_schema_version != EVALUATION_SCHEMA_VERSION
425 || self.response_schema_version != EVALUATION_SCHEMA_VERSION
426 || !valid_id(&self.dataset_id)
427 || !valid_id(&self.provider)
428 || !self.question_schema_hash.starts_with("fnv1a64:")
429 || self.question_schema_hash.len() != 24
430 || !self.fixture_content_hash.starts_with("fnv1a64:")
431 || self.fixture_content_hash.len() != 24
432 {
433 return Err(EvaluationError::Invalid("report header is invalid"));
434 }
435 if self
436 .completeness
437 .completed_cases
438 .saturating_add(self.completeness.failed_cases)
439 .saturating_add(self.completeness.missing_cases)
440 != self.completeness.total_cases
441 || self.completeness.complete
442 != (self.completeness.completed_cases == self.completeness.total_cases)
443 {
444 return Err(EvaluationError::Invalid("report completeness is invalid"));
445 }
446 for metric in &self.metrics {
447 if !valid_id(&metric.question_id)
448 || metric.total == 0
449 || metric.observed > metric.total
450 || metric.correct > metric.observed
451 || metric.accuracy.is_some_and(|value| !is_probability(value))
452 || metric.acceptance_curve.iter().any(|point| {
453 !is_probability(point.threshold)
454 || !is_probability(point.accepted_accuracy)
455 || !is_probability(point.coverage)
456 || !is_probability(point.abstention_rate)
457 || !is_probability(point.wilson_lower)
458 || !is_probability(point.wilson_upper)
459 || point.accepted_count > metric.observed
460 || point.accepted_correct > point.accepted_count
461 || point.abstention_count
462 != metric.total.saturating_sub(point.accepted_count)
463 })
464 {
465 return Err(EvaluationError::Invalid("report metric is invalid"));
466 }
467 }
468 Ok(())
469 }
470}
471
472#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
474#[serde(rename_all = "snake_case")]
475pub enum CalibrationStatus {
476 Recommended,
478 InsufficientEvidence,
480 TargetUnmet,
482 Incomplete,
484}
485
486#[derive(Debug, Clone, Serialize, Deserialize)]
488#[serde(rename_all = "camelCase", deny_unknown_fields)]
489pub struct CalibrationResult {
490 pub status: CalibrationStatus,
492 pub question_id: String,
494 pub target_accuracy: f64,
496 pub sample_size: u64,
498 #[serde(skip_serializing_if = "Option::is_none")]
500 pub threshold: Option<f64>,
501 pub accepted_count: u64,
503 pub coverage: f64,
505 #[serde(skip_serializing_if = "Option::is_none")]
507 pub accepted_accuracy: Option<f64>,
508 #[serde(skip_serializing_if = "Option::is_none")]
510 pub wilson_lower: Option<f64>,
511 #[serde(skip_serializing_if = "Option::is_none")]
513 pub wilson_upper: Option<f64>,
514}
515
516#[derive(Debug, Clone, Serialize, Deserialize)]
518#[serde(rename_all = "camelCase", deny_unknown_fields)]
519pub struct MetricRegression {
520 pub question_id: String,
522 #[serde(skip_serializing_if = "Option::is_none")]
524 pub language: Option<String>,
525 #[serde(skip_serializing_if = "Option::is_none")]
527 pub risk: Option<String>,
528 pub metric: String,
530 pub baseline: f64,
532 pub candidate: f64,
534}
535
536#[derive(Debug, Clone, Serialize, Deserialize)]
538#[serde(rename_all = "camelCase", deny_unknown_fields)]
539pub struct ComparisonReport {
540 pub comparable: bool,
542 pub changes: Vec<String>,
544 pub regressions: Vec<MetricRegression>,
546}
547
548fn is_probability(value: f64) -> bool {
549 value.is_finite() && (0.0..=1.0).contains(&value)
550}
551
552type MetricKey = (String, Option<String>, Option<String>);
553
554#[derive(Debug)]
555struct MetricAccumulator {
556 metric: QuestionMetrics,
557 observations: Vec<(f64, bool)>,
558}
559
560#[allow(
566 clippy::too_many_lines,
567 reason = "Single pass keeps case and question accounting together"
568)]
569pub fn evaluate(
570 fixture: &FixtureSuite,
571 saved: &SavedResponses,
572) -> Result<EvaluationReport, EvaluationError> {
573 fixture.validate()?;
574 saved.validate_against(fixture)?;
575 let by_case: BTreeMap<&str, &SavedOutcome> = saved
576 .results
577 .iter()
578 .map(|result| (result.case_id.as_str(), &result.outcome))
579 .collect();
580 let mut metrics: BTreeMap<MetricKey, MetricAccumulator> = BTreeMap::new();
581 let mut models = BTreeSet::new();
582 let mut latencies = Vec::new();
583 let mut completed = 0_u64;
584 let mut failed = 0_u64;
585 let mut missing = 0_u64;
586 let mut input_tokens = 0_u64;
587 let mut output_tokens = 0_u64;
588
589 for case in &fixture.cases {
590 let outcome = by_case.get(case.id.as_str()).copied();
591 let response = match outcome {
592 Some(SavedOutcome::Ok {
593 response,
594 latency_ms,
595 }) => {
596 completed = completed.saturating_add(1);
597 models.insert(response.model.clone());
598 latencies.push(*latency_ms);
599 input_tokens = input_tokens.saturating_add(response.usage.input_tokens);
600 output_tokens = output_tokens.saturating_add(response.usage.output_tokens);
601 Some(response)
602 }
603 Some(SavedOutcome::Error { .. }) => {
604 failed = failed.saturating_add(1);
605 None
606 }
607 None => {
608 missing = missing.saturating_add(1);
609 None
610 }
611 };
612 for (id, expected) in &case.expected {
613 let observed = response.and_then(|value| value.answers.get(id));
614 let question = case
615 .request
616 .questions
617 .get(id)
618 .ok_or(EvaluationError::Invalid("question missing"))?;
619 let truth = expected_label(expected);
620 let prediction = observed
621 .map(|answer| predicted_label(question, answer))
622 .transpose()?;
623 for (language, risk) in [
624 (None, None),
625 (Some(case.language.as_str()), None),
626 (None, Some(case.risk.as_str())),
627 (Some(case.language.as_str()), Some(case.risk.as_str())),
628 ] {
629 let key = (
630 id.clone(),
631 language.map(str::to_string),
632 risk.map(str::to_string),
633 );
634 let entry = metrics.entry(key).or_insert_with(|| MetricAccumulator {
635 metric: QuestionMetrics::empty(id, language, risk),
636 observations: Vec::new(),
637 });
638 entry.metric.total = entry.metric.total.saturating_add(1);
639 if let Some((label, confidence)) = &prediction {
640 entry.metric.observed = entry.metric.observed.saturating_add(1);
641 let correct = label == &truth;
642 if correct {
643 entry.metric.correct = entry.metric.correct.saturating_add(1);
644 }
645 *entry
646 .metric
647 .confusion
648 .entry(truth.clone())
649 .or_default()
650 .entry(label.clone())
651 .or_default() += 1;
652 entry.observations.push((*confidence, correct));
653 }
654 }
655 }
656 }
657
658 let metrics = metrics.into_values().map(finalize_metric).collect();
659 latencies.sort_unstable();
660 let latency_p50_ms = if latencies.is_empty() {
661 None
662 } else {
663 let middle = latencies.len() / 2;
664 if latencies.len() % 2 == 0 {
665 let left = latencies
666 .get(middle.saturating_sub(1))
667 .copied()
668 .unwrap_or(0);
669 let right = latencies.get(middle).copied().unwrap_or(0);
670 Some(f64::midpoint(to_f64(left), to_f64(right)))
671 } else {
672 latencies.get(middle).copied().map(to_f64)
673 }
674 };
675 let latency_p95_ms = if latencies.is_empty() {
676 None
677 } else {
678 let rank = 95_usize
679 .saturating_mul(latencies.len())
680 .div_ceil(100)
681 .saturating_sub(1);
682 latencies.get(rank).copied()
683 };
684 let total_cases = u64::try_from(fixture.cases.len()).unwrap_or(u64::MAX);
685 let report = EvaluationReport {
686 schema_version: EVALUATION_SCHEMA_VERSION,
687 fixture_schema_version: fixture.schema_version,
688 response_schema_version: saved.schema_version,
689 dataset_id: fixture.dataset_id.clone(),
690 provider: saved.provider.clone(),
691 models: models.into_iter().collect(),
692 question_schema_hash: question_schema_hash(fixture)?,
693 fixture_content_hash: content_hash(fixture)?,
694 completeness: Completeness {
695 total_cases,
696 completed_cases: completed,
697 failed_cases: failed,
698 missing_cases: missing,
699 complete: completed == total_cases,
700 },
701 metrics,
702 total_input_tokens: input_tokens,
703 total_output_tokens: output_tokens,
704 total_input_cost_usd: saved
705 .input_price_per_million_usd
706 .map(|price| price * to_f64(input_tokens) / 1_000_000.0),
707 total_output_cost_usd: saved
708 .output_price_per_million_usd
709 .map(|price| price * to_f64(output_tokens) / 1_000_000.0),
710 total_cost_usd: saved
711 .input_price_per_million_usd
712 .zip(saved.output_price_per_million_usd)
713 .map(|(input_price, output_price)| {
714 (input_price * to_f64(input_tokens) + output_price * to_f64(output_tokens))
715 / 1_000_000.0
716 }),
717 latency_p50_ms,
718 latency_p95_ms,
719 };
720 report.validate()?;
721 Ok(report)
722}
723
724fn expected_label(answer: &ExpectedAnswer) -> String {
725 match answer {
726 ExpectedAnswer::Noul { yes } => if *yes { "yes" } else { "no" }.to_string(),
727 ExpectedAnswer::Choice { choice } => choice.clone(),
728 ExpectedAnswer::Score { level } => level.to_string(),
729 }
730}
731
732fn predicted_label(
733 question: &Question,
734 answer: &TypedAnswer,
735) -> Result<(String, f64), EvaluationError> {
736 match (question, answer) {
737 (Question::Noul { .. }, TypedAnswer::Noul { noul }) => Ok((
738 if *noul >= 0.5 { "yes" } else { "no" }.to_string(),
739 noul.max(1.0 - noul),
740 )),
741 (
742 Question::Choice { .. },
743 TypedAnswer::Choice {
744 choice, confidence, ..
745 },
746 ) => Ok((choice.clone(), *confidence)),
747 (
748 Question::Score { criteria, .. },
749 TypedAnswer::Score {
750 score, confidence, ..
751 },
752 ) => {
753 let index = (0..criteria.len())
754 .find(|level| *score < f64::from(u32::try_from(*level).unwrap_or(u32::MAX)) + 0.5)
755 .unwrap_or_else(|| criteria.len().saturating_sub(1));
756 Ok((index.to_string(), *confidence))
757 }
758 _ => Err(EvaluationError::Invalid("answer type is incompatible")),
759 }
760}
761
762fn finalize_metric(mut accumulator: MetricAccumulator) -> QuestionMetrics {
763 let metric = &mut accumulator.metric;
764 if metric.observed > 0 {
765 metric.accuracy = Some(to_f64(metric.correct) / to_f64(metric.observed));
766 }
767 let labels: BTreeSet<String> = metric
768 .confusion
769 .iter()
770 .flat_map(|(truth, predictions)| {
771 std::iter::once(truth.clone()).chain(predictions.keys().cloned())
772 })
773 .collect();
774 for label in labels {
775 let true_count = metric
776 .confusion
777 .get(&label)
778 .map_or(0, |row| row.values().sum::<u64>());
779 let predicted_count = metric
780 .confusion
781 .values()
782 .map(|row| row.get(&label).copied().unwrap_or(0))
783 .sum::<u64>();
784 let true_positive = metric
785 .confusion
786 .get(&label)
787 .and_then(|row| row.get(&label))
788 .copied()
789 .unwrap_or(0);
790 metric
791 .false_negatives
792 .insert(label.clone(), true_count.saturating_sub(true_positive));
793 metric
794 .false_positives
795 .insert(label, predicted_count.saturating_sub(true_positive));
796 }
797 accumulator
798 .observations
799 .sort_by(|left, right| right.0.total_cmp(&left.0));
800 let mut accepted = 0_u64;
801 let mut correct = 0_u64;
802 for (index, (confidence, is_correct)) in accumulator.observations.iter().enumerate() {
803 accepted = accepted.saturating_add(1);
804 if *is_correct {
805 correct = correct.saturating_add(1);
806 }
807 let next_confidence = accumulator.observations.get(index + 1).map(|next| next.0);
808 if next_confidence.is_none_or(|next| next.total_cmp(confidence).is_ne()) {
809 let (lower, upper) = wilson_interval(correct, accepted);
810 metric.acceptance_curve.push(AcceptancePoint {
811 threshold: *confidence,
812 accepted_count: accepted,
813 accepted_correct: correct,
814 accepted_accuracy: to_f64(correct) / to_f64(accepted),
815 coverage: to_f64(accepted) / to_f64(metric.total),
816 abstention_count: metric.total.saturating_sub(accepted),
817 abstention_rate: to_f64(metric.total.saturating_sub(accepted))
818 / to_f64(metric.total),
819 wilson_lower: lower,
820 wilson_upper: upper,
821 });
822 }
823 }
824 accumulator.metric
825}
826
827fn wilson_interval(correct: u64, total: u64) -> (f64, f64) {
828 if total == 0 {
829 return (0.0, 1.0);
830 }
831 let n = to_f64(total);
832 let proportion = to_f64(correct) / n;
833 let z = 1.96_f64;
834 let z_squared = z * z;
835 let denominator = 1.0 + z_squared / n;
836 let center = (proportion + z_squared / (2.0 * n)) / denominator;
837 let margin =
838 z * ((proportion * (1.0 - proportion) + z_squared / (4.0 * n)) / n).sqrt() / denominator;
839 ((center - margin).max(0.0), (center + margin).min(1.0))
840}
841
842fn question_schema_hash(fixture: &FixtureSuite) -> Result<String, EvaluationError> {
843 let mut schemas: Vec<(&str, &BTreeMap<String, Question>)> = fixture
844 .cases
845 .iter()
846 .map(|case| (case.id.as_str(), &case.request.questions))
847 .collect();
848 schemas.sort_by_key(|(id, _)| *id);
849 let bytes = serde_json::to_vec(&schemas)
850 .map_err(|_| EvaluationError::Invalid("question schema cannot be serialized"))?;
851 Ok(fnv1a64(&bytes))
852}
853
854fn content_hash(fixture: &FixtureSuite) -> Result<String, EvaluationError> {
855 let mut cases = fixture.cases.clone();
856 cases.sort_by(|left, right| left.id.cmp(&right.id));
857 let bytes = serde_json::to_vec(&cases)
858 .map_err(|_| EvaluationError::Invalid("fixture cannot be serialized"))?;
859 Ok(fnv1a64(&bytes))
860}
861
862fn fnv1a64(bytes: &[u8]) -> String {
863 let hash = bytes.iter().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
864 (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3)
865 });
866 format!("fnv1a64:{hash:016x}")
867}
868
869#[allow(
870 clippy::cast_precision_loss,
871 reason = "Evaluation counts are bounded to 500 cases; token estimates tolerate rounding"
872)]
873fn to_f64(value: u64) -> f64 {
874 value as f64
875}
876
877pub fn calibrate(
886 report: &EvaluationReport,
887 question: &str,
888 language: Option<&str>,
889 risk: Option<&str>,
890 target_accuracy: f64,
891) -> Result<CalibrationResult, EvaluationError> {
892 report.validate()?;
893 if !target_accuracy.is_finite() || !(0.0..=1.0).contains(&target_accuracy) {
894 return Err(EvaluationError::Invalid("target accuracy is invalid"));
895 }
896 let metric = report
897 .metric_for(question, language, risk)
898 .ok_or(EvaluationError::NotFound)?;
899 let mut result = CalibrationResult {
900 status: CalibrationStatus::InsufficientEvidence,
901 question_id: question.to_string(),
902 target_accuracy,
903 sample_size: metric.total,
904 threshold: None,
905 accepted_count: 0,
906 coverage: 0.0,
907 accepted_accuracy: None,
908 wilson_lower: None,
909 wilson_upper: None,
910 };
911 if !report.completeness.complete || metric.observed != metric.total {
912 result.status = CalibrationStatus::Incomplete;
913 return Ok(result);
914 }
915 if metric.total < MIN_CALIBRATION_CASES {
916 return Ok(result);
917 }
918 let enough: Vec<&AcceptancePoint> = metric
919 .acceptance_curve
920 .iter()
921 .filter(|point| point.accepted_count >= MIN_CALIBRATION_CASES)
922 .collect();
923 if enough.is_empty() {
924 return Ok(result);
925 }
926 if let Some(point) = enough
927 .into_iter()
928 .filter(|point| point.wilson_lower >= target_accuracy)
929 .max_by_key(|point| point.accepted_count)
930 {
931 result.status = CalibrationStatus::Recommended;
932 result.threshold = Some(point.threshold);
933 result.accepted_count = point.accepted_count;
934 result.coverage = point.coverage;
935 result.accepted_accuracy = Some(point.accepted_accuracy);
936 result.wilson_lower = Some(point.wilson_lower);
937 result.wilson_upper = Some(point.wilson_upper);
938 } else {
939 result.status = CalibrationStatus::TargetUnmet;
940 }
941 Ok(result)
942}
943
944pub fn compare(
954 baseline: &EvaluationReport,
955 candidate: &EvaluationReport,
956) -> Result<ComparisonReport, EvaluationError> {
957 baseline.validate()?;
958 candidate.validate()?;
959 let mut changes = Vec::new();
960 let mut comparable = true;
961 if baseline.dataset_id != candidate.dataset_id {
962 changes.push("dataset changed".to_string());
963 comparable = false;
964 }
965 if baseline.question_schema_hash != candidate.question_schema_hash {
966 changes.push("question schema changed".to_string());
967 comparable = false;
968 }
969 if baseline.fixture_content_hash != candidate.fixture_content_hash {
970 changes.push("fixture content changed".to_string());
971 comparable = false;
972 }
973 if baseline.provider != candidate.provider {
974 changes.push("provider changed".to_string());
975 }
976 if baseline.models != candidate.models {
977 changes.push("model version changed".to_string());
978 }
979 if !baseline.completeness.complete || !candidate.completeness.complete {
980 changes.push("one or both reports are incomplete".to_string());
981 comparable = false;
982 }
983 let mut regressions = Vec::new();
984 if comparable {
985 for old in &baseline.metrics {
986 let Some(new) = candidate.metric_for(
987 &old.question_id,
988 old.language.as_deref(),
989 old.risk.as_deref(),
990 ) else {
991 changes.push("metric group changed".to_string());
992 comparable = false;
993 break;
994 };
995 if old.total != new.total {
996 changes.push("metric sample count changed".to_string());
997 comparable = false;
998 break;
999 }
1000 if let (Some(old_accuracy), Some(new_accuracy)) = (old.accuracy, new.accuracy)
1001 && new_accuracy < old_accuracy
1002 {
1003 regressions.push(MetricRegression {
1004 question_id: old.question_id.clone(),
1005 language: old.language.clone(),
1006 risk: old.risk.clone(),
1007 metric: "accuracy".to_string(),
1008 baseline: old_accuracy,
1009 candidate: new_accuracy,
1010 });
1011 }
1012 for point in &old.acceptance_curve {
1013 let candidate_accepted = new
1014 .acceptance_curve
1015 .iter()
1016 .filter(|candidate_point| candidate_point.threshold >= point.threshold)
1017 .map(|candidate_point| candidate_point.accepted_count)
1018 .max()
1019 .unwrap_or(0);
1020 let candidate_coverage = to_f64(candidate_accepted) / to_f64(new.total);
1021 if candidate_coverage < point.coverage {
1022 regressions.push(MetricRegression {
1023 question_id: old.question_id.clone(),
1024 language: old.language.clone(),
1025 risk: old.risk.clone(),
1026 metric: format!("coverage_at_threshold:{}", point.threshold),
1027 baseline: point.coverage,
1028 candidate: candidate_coverage,
1029 });
1030 }
1031 }
1032 for (label, old_count) in &old.false_negatives {
1033 let new_count = new.false_negatives.get(label).copied().unwrap_or(0);
1034 if new_count > *old_count {
1035 regressions.push(MetricRegression {
1036 question_id: old.question_id.clone(),
1037 language: old.language.clone(),
1038 risk: old.risk.clone(),
1039 metric: format!("false_negatives:{label}"),
1040 baseline: to_f64(*old_count),
1041 candidate: to_f64(new_count),
1042 });
1043 }
1044 }
1045 }
1046 }
1047 if !comparable {
1048 regressions.clear();
1049 }
1050 Ok(ComparisonReport {
1051 comparable,
1052 changes,
1053 regressions,
1054 })
1055}
1056
1057#[cfg(test)]
1058mod tests {
1059 use serde_json::json;
1060
1061 use super::{CalibrationStatus, FixtureSuite, SavedResponses, calibrate, compare, evaluate};
1062
1063 fn fixture() -> FixtureSuite {
1064 serde_json::from_value(json!({
1065 "schemaVersion": 1,
1066 "datasetId": "synthetic_mixed_v1",
1067 "cases": [{
1068 "id": "case_1", "language": "english", "risk": "high", "source": "synthetic",
1069 "request": {
1070 "state": {"title": "Existing command fails"},
1071 "questions": {
1072 "broken": {"type": "noul", "instructions": "Is the existing behavior broken?"},
1073 "kind": {"type": "choice", "instructions": "Classify the issue", "criteria": {"bug": "Broken behavior", "feature": "New behavior"}},
1074 "severity": {"type": "score", "instructions": "Rate impact", "criteria": ["Low impact", "Medium impact", "High impact"]}
1075 }
1076 },
1077 "expected": {
1078 "broken": {"type": "noul", "yes": true},
1079 "kind": {"type": "choice", "choice": "bug"},
1080 "severity": {"type": "score", "level": 2}
1081 }
1082 }]
1083 })).unwrap()
1084 }
1085
1086 fn responses() -> SavedResponses {
1087 serde_json::from_value(json!({
1088 "schemaVersion": 1, "datasetId": "synthetic_mixed_v1",
1089 "provider": "saved_fixture", "inputPricePerMillionUsd": 0.042, "outputPricePerMillionUsd": 0.3,
1090 "results": [{"caseId": "case_1", "outcome": {
1091 "status": "ok", "latencyMs": 120,
1092 "response": {
1093 "model": "fixture-model-1",
1094 "answers": {
1095 "broken": {"type": "noul", "noul": 0.9},
1096 "kind": {"type": "choice", "choice": "bug", "confidence": 0.9, "probabilities": {"bug": 0.9, "feature": 0.1}},
1097 "severity": {"type": "score", "score": 1.8, "confidence": 0.8, "legend": {"0": "Low impact", "1": "Medium impact", "2": "High impact"}, "probabilities": {"0": 0.0, "1": 0.2, "2": 0.8}}
1098 },
1099 "usage": {"input_tokens": 100, "output_tokens": 20}
1100 }
1101 }}]
1102 })).unwrap()
1103 }
1104
1105 #[test]
1106 fn test_should_evaluate_mixed_questions_offline() {
1107 let report = evaluate(&fixture(), &responses()).unwrap();
1108 assert_eq!(report.completeness.completed_cases, 1);
1109 assert_eq!(report.models, vec!["fixture-model-1"]);
1110 assert_eq!(report.total_input_tokens, 100);
1111 assert!((report.total_cost_usd.unwrap() - 0.000_010_2).abs() < 1e-12);
1112 for question in ["broken", "kind", "severity"] {
1113 let metric = report.metric_for(question, None, None).unwrap();
1114 assert_eq!(metric.correct, 1);
1115 assert_eq!(metric.observed, 1);
1116 }
1117 assert!(
1118 report
1119 .metric_for("kind", Some("english"), Some("high"))
1120 .is_some()
1121 );
1122 }
1123
1124 #[test]
1125 fn test_should_reject_sensitive_fixture() {
1126 let mut suite = fixture();
1127 suite.cases[0].request.state =
1128 json!({"title": "See https://internal.example.local/private"});
1129 assert!(suite.validate().is_err());
1130 suite.cases[0].request.state = json!({"title": format!("{}{}", "apikey_", "x".repeat(40))});
1131 assert!(suite.validate().is_err());
1132 suite = fixture();
1133 if let crate::decision::Question::Noul { instructions } =
1134 suite.cases[0].request.questions.get_mut("broken").unwrap()
1135 {
1136 *instructions = format!("Check {}{}", "apikey_", "x".repeat(40));
1137 }
1138 assert!(suite.validate().is_err());
1139 }
1140
1141 #[test]
1142 fn test_should_mark_missing_response_incomplete() {
1143 let mut saved = responses();
1144 saved.results.clear();
1145 let report = evaluate(&fixture(), &saved).unwrap();
1146 assert_eq!(report.completeness.missing_cases, 1);
1147 assert!(!report.completeness.complete);
1148 assert_eq!(report.metric_for("kind", None, None).unwrap().observed, 0);
1149 }
1150
1151 #[test]
1152 fn test_should_refuse_calibration_on_small_sample() {
1153 let report = evaluate(&fixture(), &responses()).unwrap();
1154 let result = calibrate(&report, "kind", None, None, 0.8).unwrap();
1155 assert_eq!(result.status, CalibrationStatus::InsufficientEvidence);
1156 assert!(result.threshold.is_none());
1157 }
1158
1159 #[test]
1160 fn test_should_recommend_only_when_wilson_bound_meets_target() {
1161 let mut suite = fixture();
1162 let mut saved = responses();
1163 for index in 2..=40 {
1164 let id = format!("case_{index}");
1165 let mut case = suite.cases[0].clone();
1166 case.id.clone_from(&id);
1167 suite.cases.push(case);
1168 let mut result = saved.results[0].clone();
1169 result.case_id = id;
1170 saved.results.push(result);
1171 }
1172 let report = evaluate(&suite, &saved).unwrap();
1173 let recommended = calibrate(&report, "kind", None, None, 0.90).unwrap();
1174 assert_eq!(recommended.status, CalibrationStatus::Recommended);
1175 assert_eq!(recommended.sample_size, 40);
1176 assert_eq!(recommended.accepted_count, 40);
1177 assert!((recommended.coverage - 1.0).abs() < f64::EPSILON);
1178 assert!(recommended.wilson_lower.unwrap() >= 0.90);
1179 let unmet = calibrate(&report, "kind", None, None, 0.99).unwrap();
1180 assert_eq!(unmet.status, CalibrationStatus::TargetUnmet);
1181 }
1182
1183 #[test]
1184 fn test_should_detect_schema_change_and_metric_regression() {
1185 let baseline = evaluate(&fixture(), &responses()).unwrap();
1186 let mut candidate = baseline.clone();
1187 candidate.question_schema_hash = "fnv1a64:0000000000000000".into();
1188 assert!(!compare(&baseline, &candidate).unwrap().comparable);
1189 candidate.question_schema_hash = baseline.question_schema_hash.clone();
1190 candidate.metrics[0].correct = 0;
1191 candidate.metrics[0].accuracy = Some(0.0);
1192 assert!(
1193 !compare(&baseline, &candidate)
1194 .unwrap()
1195 .regressions
1196 .is_empty()
1197 );
1198 }
1199
1200 #[test]
1201 fn test_should_detect_fixture_content_change_with_same_dataset_id() {
1202 let baseline = evaluate(&fixture(), &responses()).unwrap();
1203 let mut changed = fixture();
1204 changed.cases[0].request.state = json!({"title": "A different public issue"});
1205 let candidate = evaluate(&changed, &responses()).unwrap();
1206 let comparison = compare(&baseline, &candidate).unwrap();
1207 assert!(!comparison.comparable);
1208 assert!(
1209 comparison
1210 .changes
1211 .iter()
1212 .any(|change| change == "fixture content changed")
1213 );
1214 }
1215
1216 #[test]
1217 fn test_should_detect_coverage_regression_at_baseline_threshold() {
1218 let baseline = evaluate(&fixture(), &responses()).unwrap();
1219 let mut changed = responses();
1220 if let super::SavedOutcome::Ok { response, .. } = &mut changed.results[0].outcome
1221 && let crate::decision::TypedAnswer::Choice {
1222 confidence,
1223 probabilities,
1224 ..
1225 } = response.answers.get_mut("kind").unwrap()
1226 {
1227 *confidence = 0.6;
1228 probabilities.insert("bug".into(), 0.6);
1229 probabilities.insert("feature".into(), 0.4);
1230 }
1231 let candidate = evaluate(&fixture(), &changed).unwrap();
1232 let comparison = compare(&baseline, &candidate).unwrap();
1233 assert!(comparison.comparable);
1234 assert!(
1235 comparison
1236 .regressions
1237 .iter()
1238 .any(|regression| regression.question_id == "kind"
1239 && regression.metric.starts_with("coverage_at_threshold:"))
1240 );
1241 }
1242}