Skip to main content

firstpass_proxy/
consistency.rs

1//! Self-consistency uncertainty gate: measure a model's own confidence by resampling.
2//!
3//! **Research basis**
4//! - Wang et al. 2022 ("Self-Consistency Improves Chain of Thought Reasoning in Language
5//!   Models"): sampling multiple reasoning paths and taking the most-consistent answer
6//!   significantly improves accuracy over greedy decoding. Answers a model reliably reproduces
7//!   are far likelier correct.
8//! - Farquhar et al., *Nature* 2024 ("Detecting Hallucinations in Large Language Models Using
9//!   Semantic Entropy"): disagreement across samples — semantic entropy — flags hallucination.
10//!   A continuous agreement score lets the conformal threshold machinery calibrate the serve
11//!   cutoff against a target failure rate.
12//!
13//! **Mechanism**: call the same model `k` times on the original request (concurrently — serial
14//! would multiply added latency by k), then score the candidate's agreement with each sample.
15//! Per-sample agreement: (a) exact final-answer match (1.0 if the last number in the text, or
16//! the whole normalized text, matches) or (b) token-set Jaccard similarity. Gate score is the
17//! mean over all usable samples, clamped `[0, 1]`. Verdict = Pass iff score ≥ threshold.
18//!
19//! **maker == checker is ALLOWED** — unlike [`crate::judge::JudgeGate`], self-consistency is
20//! definitionally self-referential: the model samples *itself*. This is not a bug; it is the
21//! mechanism. The gate enforces no maker ≠ checker constraint.
22//!
23//! **Honest scope**
24//! - ponytail: lexical agreement (final-number extraction + token-set Jaccard) is a cheap proxy
25//!   for semantic clustering. Strongest on short, factual, or structured outputs (numbers,
26//!   labels, code identifiers). Upgrade to entailment-based semantic clustering via a judge
27//!   model for long-form or heavily paraphrastic outputs.
28//! - Cost: k extra model calls per request — that is the honest price of measured confidence.
29//!   Pair with a cheap model (e.g. haiku) on the first rung so the sampling cost stays low;
30//!   expensive frontier models only bear escalation cost when cheaper rungs fail.
31
32use std::sync::Arc;
33use std::time::Instant;
34
35use async_trait::async_trait;
36use firstpass_core::{GateResult, Score, Verdict, cost::PriceTable};
37
38use crate::gate::Gate;
39use crate::provider::{Auth, ModelRequest, ModelResponse, Provider};
40
41/// A gate that scores the candidate by agreement with `k` fresh samples of the same model.
42#[derive(Debug)]
43pub struct ConsistencyGate {
44    id: String,
45    provider: Arc<dyn Provider>,
46    /// `provider/model` used for resampling. May equal the candidate's model — expected.
47    sample_model: String,
48    auth: Auth,
49    k: u32,
50    threshold: f64,
51    /// Prices the k sample calls so their cost lands on the receipt ([`GateResult::cost_usd`]).
52    prices: PriceTable,
53}
54
55impl ConsistencyGate {
56    /// Build a consistency gate. `provider` serves `sample_model`; `auth` carries credentials;
57    /// `prices` prices the k sample calls onto the receipt.
58    #[must_use]
59    pub fn new(
60        id: impl Into<String>,
61        provider: Arc<dyn Provider>,
62        sample_model: impl Into<String>,
63        auth: Auth,
64        k: u32,
65        threshold: f64,
66        prices: PriceTable,
67    ) -> Self {
68        Self {
69            id: id.into(),
70            provider,
71            sample_model: sample_model.into(),
72            auth,
73            k,
74            threshold,
75            prices,
76        }
77    }
78}
79
80#[async_trait]
81impl Gate for ConsistencyGate {
82    fn id(&self) -> &str {
83        &self.id
84    }
85
86    async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
87        let start = Instant::now();
88
89        // Build the sample request: same conversation, model swapped to the configured sampler.
90        let mut sample_req = req.clone();
91        sample_req.model = self.sample_model.clone();
92
93        // Fire k sample calls concurrently — serial would multiply added latency by k.
94        let mut set = tokio::task::JoinSet::new();
95        for _ in 0..self.k {
96            let provider = Arc::clone(&self.provider);
97            let req_clone = sample_req.clone();
98            let auth_clone = self.auth.clone();
99            set.spawn(async move { provider.complete(&req_clone, &auth_clone).await });
100        }
101
102        let mut samples: Vec<String> = Vec::new();
103        let mut sample_cost = 0.0_f64;
104        while let Some(res) = set.join_next().await {
105            // Provider error or task panic → drop this sample, score over the rest.
106            if let Ok(Ok(sample)) = res {
107                // Each sample is a real model call — price it onto the receipt so the router's
108                // budget/savings math sees the true cost of measured confidence.
109                // Cache-aware: consistency re-sends one prompt k times, which is precisely the
110                // shape prompt caching is built for — so on a cached prompt nearly all of this
111                // spend lands in the cache counters and `in_tokens` alone reports close to zero.
112                sample_cost += self
113                    .prices
114                    .cost_usd_with_cache(
115                        &self.sample_model,
116                        sample.in_tokens,
117                        sample.cache_write_tokens,
118                        sample.cache_read_tokens,
119                        sample.out_tokens,
120                    )
121                    .unwrap_or(0.0);
122                samples.push(sample.text);
123            }
124        }
125
126        let ms = elapsed_ms(start);
127
128        // Zero usable samples → abstain; never fabricate a score.
129        if samples.is_empty() {
130            let mut r = GateResult::abstain(&self.id, "no_usable_samples", ms);
131            r.evidence_ref = Some(format!("all {} sample calls failed", self.k));
132            r.cost_usd = sample_cost;
133            return r;
134        }
135
136        let candidate_norm = normalize(&resp.text);
137        let candidate_answer = extract_final_answer(&candidate_norm);
138
139        let total: f64 = samples
140            .iter()
141            .map(|s| {
142                let snorm = normalize(s);
143                let sanswer = extract_final_answer(&snorm);
144                if candidate_answer == sanswer {
145                    1.0_f64
146                } else {
147                    // ponytail: token-set Jaccard over full normalized text. Upgrade to
148                    // entailment-based clustering via a judge model for semantic agreement on
149                    // long-form or paraphrastic outputs.
150                    jaccard(&candidate_norm, &snorm)
151                }
152            })
153            .sum();
154
155        let score_val = (total / samples.len() as f64).clamp(0.0, 1.0);
156        let verdict = if score_val >= self.threshold {
157            Verdict::Pass
158        } else {
159            Verdict::Fail
160        };
161
162        let mut r = GateResult::deterministic(&self.id, verdict, ms);
163        r.score = Score::new(score_val).ok();
164        r.cost_usd = sample_cost;
165        r
166    }
167}
168
169/// Normalize text for comparison: trim, lowercase, collapse internal whitespace to single spaces.
170fn normalize(text: &str) -> String {
171    text.trim()
172        .to_lowercase()
173        .split_whitespace()
174        .collect::<Vec<_>>()
175        .join(" ")
176}
177
178/// Extract the final answer from normalized text: the last contiguous ASCII-digit sequence,
179/// or the whole string when no digits are present.
180///
181/// This is the cheap "what number did the model settle on?" proxy from self-consistency
182/// literature. Strongest on single-number answers (arithmetic, retrieval).
183///
184// ponytail: digit-only extraction. Upgrade to full expression parsing if answers include
185// decimals, fractions, or labelled fields (e.g. "answer: 42.5").
186fn extract_final_answer(normalized: &str) -> String {
187    let mut last_num = String::new();
188    let mut current_num = String::new();
189    for c in normalized.chars() {
190        if c.is_ascii_digit() {
191            current_num.push(c);
192        } else if !current_num.is_empty() {
193            last_num.clone_from(&current_num);
194            current_num.clear();
195        }
196    }
197    if !current_num.is_empty() {
198        last_num = current_num;
199    }
200    if last_num.is_empty() {
201        normalized.to_owned()
202    } else {
203        last_num
204    }
205}
206
207/// Token-set Jaccard similarity: `|A ∩ B| / |A ∪ B|`, tokens split on whitespace.
208/// Two empty strings → 1.0 (identical); one empty vs one non-empty → 0.0.
209fn jaccard(a: &str, b: &str) -> f64 {
210    let tokens_a: std::collections::HashSet<&str> = a.split_whitespace().collect();
211    let tokens_b: std::collections::HashSet<&str> = b.split_whitespace().collect();
212    let inter = tokens_a.intersection(&tokens_b).count();
213    let union = tokens_a.len() + tokens_b.len() - inter;
214    if union == 0 {
215        1.0 // both empty → identical
216    } else {
217        inter as f64 / union as f64
218    }
219}
220
221fn elapsed_ms(start: Instant) -> u64 {
222    u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
223}
224
225#[cfg(test)]
226mod tests {
227    use super::*;
228    use crate::provider::{ChatMessage, MockProvider, ProviderError};
229    use serde_json::Value;
230    use std::collections::HashMap;
231    use std::sync::Arc;
232
233    fn make_req() -> ModelRequest {
234        ModelRequest {
235            model: "anthropic/claude-haiku-4-5".to_owned(),
236            system: None,
237            messages: vec![ChatMessage::text("user", "What is 6 * 7?")],
238            max_tokens: 64,
239            tools: Value::Null,
240            raw: Value::Null,
241            cache_prefix: false,
242        }
243    }
244
245    fn make_resp(model: &str, text: &str) -> ModelResponse {
246        ModelResponse {
247            model: model.to_owned(),
248            text: text.to_owned(),
249            in_tokens: 10,
250            cache_write_tokens: 0,
251            cache_read_tokens: 0,
252            out_tokens: 10,
253            raw: Value::Null,
254        }
255    }
256
257    fn provider_returning(sample_model: &str, text: &str) -> Arc<dyn Provider> {
258        let mut outcomes = HashMap::new();
259        outcomes.insert(sample_model.to_owned(), Ok(make_resp(sample_model, text)));
260        Arc::new(MockProvider::new(
261            sample_model.split('/').next().unwrap_or("mock"),
262            outcomes,
263        ))
264    }
265
266    fn provider_erroring(sample_model: &str) -> Arc<dyn Provider> {
267        let mut outcomes = HashMap::new();
268        outcomes.insert(
269            sample_model.to_owned(),
270            Err(ProviderError::Transport("down".to_owned())),
271        );
272        Arc::new(MockProvider::new(
273            sample_model.split('/').next().unwrap_or("mock"),
274            outcomes,
275        ))
276    }
277
278    /// Agreeing samples (same final answer "42", phrasing differs) → high score, Pass at 0.6.
279    #[tokio::test]
280    async fn agreeing_samples_pass_at_threshold_0_6() {
281        // k=3 samples all return "The answer is 42" while candidate says "Therefore the answer
282        // is 42." — final answers agree (both extract to "42") → score 1.0 → Pass.
283        let gate = ConsistencyGate::new(
284            "uncertainty",
285            provider_returning("anthropic/sampler", "The answer is 42"),
286            "anthropic/sampler",
287            Auth::default(),
288            3,
289            0.6,
290            PriceTable::default(),
291        );
292        let candidate = make_resp("anthropic/claude-haiku-4-5", "Therefore the answer is 42.");
293        let result = gate.evaluate(&make_req(), &candidate).await;
294        assert_eq!(result.verdict, Verdict::Pass, "final-answer match → Pass");
295        let score = result
296            .score
297            .expect("score must be present on a non-abstain verdict");
298        assert!(
299            score.value() >= 0.9,
300            "all samples agree → score near 1.0; got {score}"
301        );
302    }
303
304    /// Disagreeing samples (different numbers) → low score, Fail at threshold 0.6.
305    #[tokio::test]
306    async fn disagreeing_samples_fail_at_threshold_0_6() {
307        // k=3 samples return bare "7" while candidate says "The answer is 42".
308        // Different final answers ("7" vs "42"); Jaccard("the answer is 42", "7") = 0/5 = 0.0,
309        // well below threshold 0.6 → Fail.
310        let gate = ConsistencyGate::new(
311            "uncertainty",
312            provider_returning("anthropic/sampler", "7"),
313            "anthropic/sampler",
314            Auth::default(),
315            3,
316            0.6,
317            PriceTable::default(),
318        );
319        let candidate = make_resp("anthropic/claude-haiku-4-5", "The answer is 42");
320        let result = gate.evaluate(&make_req(), &candidate).await;
321        assert_eq!(
322            result.verdict,
323            Verdict::Fail,
324            "different final answers → Fail"
325        );
326    }
327
328    /// Exactly k calls are fired — confirming the concurrent fan-out.
329    #[tokio::test]
330    async fn exactly_k_sample_calls_are_fired() {
331        let k: u32 = 3;
332        let mut outcomes = HashMap::new();
333        outcomes.insert(
334            "anthropic/sampler".to_owned(),
335            Ok(make_resp("anthropic/sampler", "42")),
336        );
337        let mock = MockProvider::new("anthropic", outcomes);
338        let call_log = mock.call_log();
339        let gate = ConsistencyGate::new(
340            "uncertainty",
341            Arc::new(mock),
342            "anthropic/sampler",
343            Auth::default(),
344            k,
345            0.5,
346            PriceTable::default(),
347        );
348        gate.evaluate(&make_req(), &make_resp("anthropic/claude-haiku-4-5", "42"))
349            .await;
350        let log = call_log.lock().unwrap();
351        assert_eq!(log.len(), k as usize, "exactly k calls must be fired");
352        assert!(
353            log.iter().all(|m| m == "anthropic/sampler"),
354            "all calls target the configured sample model"
355        );
356    }
357
358    /// All samples error → Abstain with reason "no_usable_samples".
359    #[tokio::test]
360    async fn all_samples_error_abstains() {
361        let gate = ConsistencyGate::new(
362            "uncertainty",
363            provider_erroring("anthropic/sampler"),
364            "anthropic/sampler",
365            Auth::default(),
366            3,
367            0.6,
368            PriceTable::default(),
369        );
370        let candidate = make_resp("anthropic/claude-haiku-4-5", "42");
371        let result = gate.evaluate(&make_req(), &candidate).await;
372        assert_eq!(result.verdict, Verdict::Abstain);
373        assert_eq!(
374            result.reason.as_deref(),
375            Some("no_usable_samples"),
376            "all-error abstain carries the expected reason"
377        );
378    }
379
380    // ---- unit tests for the scoring helpers ----
381
382    #[test]
383    fn normalize_trims_lowercases_and_collapses_whitespace() {
384        assert_eq!(normalize("  Hello   World  "), "hello world");
385        assert_eq!(normalize("THE ANSWER IS 42"), "the answer is 42");
386        assert_eq!(normalize(""), "");
387    }
388
389    #[test]
390    fn extract_final_answer_returns_last_digit_sequence() {
391        assert_eq!(extract_final_answer("the answer is 42"), "42");
392        assert_eq!(extract_final_answer("3 times 7 equals 21"), "21");
393        // Trailing non-digit — still gets the last number.
394        assert_eq!(extract_final_answer("result: 100 units"), "100");
395        // No digits → whole string.
396        assert_eq!(extract_final_answer("no numbers here"), "no numbers here");
397        // Empty string.
398        assert_eq!(extract_final_answer(""), "");
399    }
400
401    #[test]
402    fn jaccard_similarity_boundary_values() {
403        // Identical tokens → 1.0.
404        assert!((jaccard("a b c", "a b c") - 1.0).abs() < 1e-9);
405        // Disjoint tokens → 0.0.
406        assert!((jaccard("a b", "c d") - 0.0).abs() < 1e-9);
407        // Half overlap: {a,b,c} ∩ {b,c,d} = {b,c}, union = 4 → 0.5.
408        let j = jaccard("a b c", "b c d");
409        assert!((j - 0.5).abs() < 1e-9, "expected 0.5, got {j}");
410        // Both empty → identical (1.0).
411        assert!((jaccard("", "") - 1.0).abs() < 1e-9);
412        // One empty → 0.0.
413        assert!((jaccard("a b", "") - 0.0).abs() < 1e-9);
414    }
415
416    #[tokio::test]
417    async fn sample_cost_lands_on_the_receipt() {
418        use firstpass_core::cost::ModelPrice;
419        // k=3 samples at 10 in / 10 out tokens each, priced 1.0/5.0 per Mtok.
420        let prices = PriceTable::new().with_override(
421            "anthropic/sampler",
422            ModelPrice {
423                input_per_mtok: 1.0,
424                output_per_mtok: 5.0,
425            },
426        );
427        let gate = ConsistencyGate::new(
428            "uncertainty",
429            provider_returning("anthropic/sampler", "42"),
430            "anthropic/sampler",
431            Auth::default(),
432            3,
433            0.5,
434            prices,
435        );
436        let r = gate
437            .evaluate(&make_req(), &make_resp("anthropic/sampler", "42"))
438            .await;
439        // 3 samples x (10/1e6*1.0 + 10/1e6*5.0) = 3 x 6e-5 = 1.8e-4.
440        let expected = 3.0 * (10.0 / 1e6 * 1.0 + 10.0 / 1e6 * 5.0);
441        assert!(
442            (r.cost_usd - expected).abs() < 1e-12,
443            "k sample calls priced onto the receipt: got {}, want {expected}",
444            r.cost_usd
445        );
446    }
447}