1use std::sync::Arc;
33use std::time::Instant;
34
35use async_trait::async_trait;
36use firstpass_core::{GateResult, Score, Verdict, cost::PriceTable};
37
38use crate::gate::Gate;
39use crate::provider::{Auth, ModelRequest, ModelResponse, Provider};
40
41#[derive(Debug)]
43pub struct ConsistencyGate {
44 id: String,
45 provider: Arc<dyn Provider>,
46 sample_model: String,
48 auth: Auth,
49 k: u32,
50 threshold: f64,
51 prices: PriceTable,
53}
54
55impl ConsistencyGate {
56 #[must_use]
59 pub fn new(
60 id: impl Into<String>,
61 provider: Arc<dyn Provider>,
62 sample_model: impl Into<String>,
63 auth: Auth,
64 k: u32,
65 threshold: f64,
66 prices: PriceTable,
67 ) -> Self {
68 Self {
69 id: id.into(),
70 provider,
71 sample_model: sample_model.into(),
72 auth,
73 k,
74 threshold,
75 prices,
76 }
77 }
78}
79
80#[async_trait]
81impl Gate for ConsistencyGate {
82 fn id(&self) -> &str {
83 &self.id
84 }
85
86 async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
87 let start = Instant::now();
88
89 let mut sample_req = req.clone();
91 sample_req.model = self.sample_model.clone();
92
93 let mut set = tokio::task::JoinSet::new();
95 for _ in 0..self.k {
96 let provider = Arc::clone(&self.provider);
97 let req_clone = sample_req.clone();
98 let auth_clone = self.auth.clone();
99 set.spawn(async move { provider.complete(&req_clone, &auth_clone).await });
100 }
101
102 let mut samples: Vec<String> = Vec::new();
103 let mut sample_cost = 0.0_f64;
104 while let Some(res) = set.join_next().await {
105 if let Ok(Ok(sample)) = res {
107 sample_cost += self
113 .prices
114 .cost_usd_with_cache(
115 &self.sample_model,
116 sample.in_tokens,
117 sample.cache_write_tokens,
118 sample.cache_read_tokens,
119 sample.out_tokens,
120 )
121 .unwrap_or(0.0);
122 samples.push(sample.text);
123 }
124 }
125
126 let ms = elapsed_ms(start);
127
128 if samples.is_empty() {
130 let mut r = GateResult::abstain(&self.id, "no_usable_samples", ms);
131 r.evidence_ref = Some(format!("all {} sample calls failed", self.k));
132 r.cost_usd = sample_cost;
133 return r;
134 }
135
136 let candidate_norm = normalize(&resp.text);
137 let candidate_answer = extract_final_answer(&candidate_norm);
138
139 let total: f64 = samples
140 .iter()
141 .map(|s| {
142 let snorm = normalize(s);
143 let sanswer = extract_final_answer(&snorm);
144 if candidate_answer == sanswer {
145 1.0_f64
146 } else {
147 jaccard(&candidate_norm, &snorm)
151 }
152 })
153 .sum();
154
155 let score_val = (total / samples.len() as f64).clamp(0.0, 1.0);
156 let verdict = if score_val >= self.threshold {
157 Verdict::Pass
158 } else {
159 Verdict::Fail
160 };
161
162 let mut r = GateResult::deterministic(&self.id, verdict, ms);
163 r.score = Score::new(score_val).ok();
164 r.cost_usd = sample_cost;
165 r
166 }
167}
168
169fn normalize(text: &str) -> String {
171 text.trim()
172 .to_lowercase()
173 .split_whitespace()
174 .collect::<Vec<_>>()
175 .join(" ")
176}
177
178fn extract_final_answer(normalized: &str) -> String {
187 let mut last_num = String::new();
188 let mut current_num = String::new();
189 for c in normalized.chars() {
190 if c.is_ascii_digit() {
191 current_num.push(c);
192 } else if !current_num.is_empty() {
193 last_num.clone_from(¤t_num);
194 current_num.clear();
195 }
196 }
197 if !current_num.is_empty() {
198 last_num = current_num;
199 }
200 if last_num.is_empty() {
201 normalized.to_owned()
202 } else {
203 last_num
204 }
205}
206
207fn jaccard(a: &str, b: &str) -> f64 {
210 let tokens_a: std::collections::HashSet<&str> = a.split_whitespace().collect();
211 let tokens_b: std::collections::HashSet<&str> = b.split_whitespace().collect();
212 let inter = tokens_a.intersection(&tokens_b).count();
213 let union = tokens_a.len() + tokens_b.len() - inter;
214 if union == 0 {
215 1.0 } else {
217 inter as f64 / union as f64
218 }
219}
220
221fn elapsed_ms(start: Instant) -> u64 {
222 u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
223}
224
225#[cfg(test)]
226mod tests {
227 use super::*;
228 use crate::provider::{ChatMessage, MockProvider, ProviderError};
229 use serde_json::Value;
230 use std::collections::HashMap;
231 use std::sync::Arc;
232
233 fn make_req() -> ModelRequest {
234 ModelRequest {
235 model: "anthropic/claude-haiku-4-5".to_owned(),
236 system: None,
237 messages: vec![ChatMessage::text("user", "What is 6 * 7?")],
238 max_tokens: 64,
239 tools: Value::Null,
240 raw: Value::Null,
241 cache_prefix: false,
242 }
243 }
244
245 fn make_resp(model: &str, text: &str) -> ModelResponse {
246 ModelResponse {
247 model: model.to_owned(),
248 text: text.to_owned(),
249 in_tokens: 10,
250 cache_write_tokens: 0,
251 cache_read_tokens: 0,
252 out_tokens: 10,
253 raw: Value::Null,
254 }
255 }
256
257 fn provider_returning(sample_model: &str, text: &str) -> Arc<dyn Provider> {
258 let mut outcomes = HashMap::new();
259 outcomes.insert(sample_model.to_owned(), Ok(make_resp(sample_model, text)));
260 Arc::new(MockProvider::new(
261 sample_model.split('/').next().unwrap_or("mock"),
262 outcomes,
263 ))
264 }
265
266 fn provider_erroring(sample_model: &str) -> Arc<dyn Provider> {
267 let mut outcomes = HashMap::new();
268 outcomes.insert(
269 sample_model.to_owned(),
270 Err(ProviderError::Transport("down".to_owned())),
271 );
272 Arc::new(MockProvider::new(
273 sample_model.split('/').next().unwrap_or("mock"),
274 outcomes,
275 ))
276 }
277
278 #[tokio::test]
280 async fn agreeing_samples_pass_at_threshold_0_6() {
281 let gate = ConsistencyGate::new(
284 "uncertainty",
285 provider_returning("anthropic/sampler", "The answer is 42"),
286 "anthropic/sampler",
287 Auth::default(),
288 3,
289 0.6,
290 PriceTable::default(),
291 );
292 let candidate = make_resp("anthropic/claude-haiku-4-5", "Therefore the answer is 42.");
293 let result = gate.evaluate(&make_req(), &candidate).await;
294 assert_eq!(result.verdict, Verdict::Pass, "final-answer match → Pass");
295 let score = result
296 .score
297 .expect("score must be present on a non-abstain verdict");
298 assert!(
299 score.value() >= 0.9,
300 "all samples agree → score near 1.0; got {score}"
301 );
302 }
303
304 #[tokio::test]
306 async fn disagreeing_samples_fail_at_threshold_0_6() {
307 let gate = ConsistencyGate::new(
311 "uncertainty",
312 provider_returning("anthropic/sampler", "7"),
313 "anthropic/sampler",
314 Auth::default(),
315 3,
316 0.6,
317 PriceTable::default(),
318 );
319 let candidate = make_resp("anthropic/claude-haiku-4-5", "The answer is 42");
320 let result = gate.evaluate(&make_req(), &candidate).await;
321 assert_eq!(
322 result.verdict,
323 Verdict::Fail,
324 "different final answers → Fail"
325 );
326 }
327
328 #[tokio::test]
330 async fn exactly_k_sample_calls_are_fired() {
331 let k: u32 = 3;
332 let mut outcomes = HashMap::new();
333 outcomes.insert(
334 "anthropic/sampler".to_owned(),
335 Ok(make_resp("anthropic/sampler", "42")),
336 );
337 let mock = MockProvider::new("anthropic", outcomes);
338 let call_log = mock.call_log();
339 let gate = ConsistencyGate::new(
340 "uncertainty",
341 Arc::new(mock),
342 "anthropic/sampler",
343 Auth::default(),
344 k,
345 0.5,
346 PriceTable::default(),
347 );
348 gate.evaluate(&make_req(), &make_resp("anthropic/claude-haiku-4-5", "42"))
349 .await;
350 let log = call_log.lock().unwrap();
351 assert_eq!(log.len(), k as usize, "exactly k calls must be fired");
352 assert!(
353 log.iter().all(|m| m == "anthropic/sampler"),
354 "all calls target the configured sample model"
355 );
356 }
357
358 #[tokio::test]
360 async fn all_samples_error_abstains() {
361 let gate = ConsistencyGate::new(
362 "uncertainty",
363 provider_erroring("anthropic/sampler"),
364 "anthropic/sampler",
365 Auth::default(),
366 3,
367 0.6,
368 PriceTable::default(),
369 );
370 let candidate = make_resp("anthropic/claude-haiku-4-5", "42");
371 let result = gate.evaluate(&make_req(), &candidate).await;
372 assert_eq!(result.verdict, Verdict::Abstain);
373 assert_eq!(
374 result.reason.as_deref(),
375 Some("no_usable_samples"),
376 "all-error abstain carries the expected reason"
377 );
378 }
379
380 #[test]
383 fn normalize_trims_lowercases_and_collapses_whitespace() {
384 assert_eq!(normalize(" Hello World "), "hello world");
385 assert_eq!(normalize("THE ANSWER IS 42"), "the answer is 42");
386 assert_eq!(normalize(""), "");
387 }
388
389 #[test]
390 fn extract_final_answer_returns_last_digit_sequence() {
391 assert_eq!(extract_final_answer("the answer is 42"), "42");
392 assert_eq!(extract_final_answer("3 times 7 equals 21"), "21");
393 assert_eq!(extract_final_answer("result: 100 units"), "100");
395 assert_eq!(extract_final_answer("no numbers here"), "no numbers here");
397 assert_eq!(extract_final_answer(""), "");
399 }
400
401 #[test]
402 fn jaccard_similarity_boundary_values() {
403 assert!((jaccard("a b c", "a b c") - 1.0).abs() < 1e-9);
405 assert!((jaccard("a b", "c d") - 0.0).abs() < 1e-9);
407 let j = jaccard("a b c", "b c d");
409 assert!((j - 0.5).abs() < 1e-9, "expected 0.5, got {j}");
410 assert!((jaccard("", "") - 1.0).abs() < 1e-9);
412 assert!((jaccard("a b", "") - 0.0).abs() < 1e-9);
414 }
415
416 #[tokio::test]
417 async fn sample_cost_lands_on_the_receipt() {
418 use firstpass_core::cost::ModelPrice;
419 let prices = PriceTable::new().with_override(
421 "anthropic/sampler",
422 ModelPrice {
423 input_per_mtok: 1.0,
424 output_per_mtok: 5.0,
425 },
426 );
427 let gate = ConsistencyGate::new(
428 "uncertainty",
429 provider_returning("anthropic/sampler", "42"),
430 "anthropic/sampler",
431 Auth::default(),
432 3,
433 0.5,
434 prices,
435 );
436 let r = gate
437 .evaluate(&make_req(), &make_resp("anthropic/sampler", "42"))
438 .await;
439 let expected = 3.0 * (10.0 / 1e6 * 1.0 + 10.0 / 1e6 * 5.0);
441 assert!(
442 (r.cost_usd - expected).abs() < 1e-12,
443 "k sample calls priced onto the receipt: got {}, want {expected}",
444 r.cost_usd
445 );
446 }
447}