use crate::error::{Result, UcciError};
use crate::num::{check_same_len, np_mean};
pub const PROBABILITY_TOLERANCE: f64 = 1e-9;
pub const MAX_LOGPROB: f64 = 9.999999995e-10;
const EMPTY_GENERATION: &str =
"generation (u(x) needs at least one generated content token; Eq. 4 averages over T >= 1)";
pub fn token_margin(a: f64, b: f64) -> Result<f64> {
margin_at(a, b, 0)
}
fn in_prob_range(p: f64) -> bool {
(0.0..=1.0 + PROBABILITY_TOLERANCE).contains(&p)
}
fn margin_at(a: f64, b: f64, index: usize) -> Result<f64> {
let (p1, p2) = if a >= b { (a, b) } else { (b, a) };
let valid = 0.0 <= p2 && p2 <= p1 && p1 <= 1.0 + PROBABILITY_TOLERANCE;
if !valid {
let value = if in_prob_range(a) { b } else { a };
return Err(UcciError::OutOfRange {
what: "top-2 probability",
index,
value,
expected: "in [0, 1]",
});
}
Ok(p1.min(1.0) - p2)
}
pub fn margins_from_top2(pairs: &[(f64, f64)]) -> Result<Vec<f64>> {
pairs
.iter()
.enumerate()
.map(|(t, &(a, b))| margin_at(a, b, t))
.collect()
}
pub fn uncertainty_from_margins(margins: &[f64]) -> Result<f64> {
if margins.is_empty() {
return Err(UcciError::Empty {
what: EMPTY_GENERATION,
});
}
crate::num::check_finite(margins, "margins")?;
if let Some(index) = margins.iter().position(|m| !(0.0..=1.0).contains(m)) {
return Err(UcciError::OutOfRange {
what: "margins",
index,
value: margins[index],
expected: "in [0, 1]",
});
}
Ok(1.0 - np_mean(margins))
}
pub fn token_margin_uncertainty(pairs: &[(f64, f64)]) -> Result<f64> {
uncertainty_from_margins(&margins_from_top2(pairs)?)
}
pub fn uncertainty_from_probs(p1: &[f64], p2: &[f64]) -> Result<f64> {
check_probabilities(p1, "p1")?;
check_probabilities(p2, "p2")?;
check_same_len(p1.len(), p2.len(), "p1 and p2")?;
if p1.is_empty() {
return Err(UcciError::Empty {
what: EMPTY_GENERATION,
});
}
let margins: Vec<f64> = p1
.iter()
.zip(p2)
.map(|(&a, &b)| a.max(b).min(1.0) - a.min(b))
.collect();
Ok(1.0 - np_mean(&margins))
}
pub fn uncertainty_from_logprobs(lp1: &[f64], lp2: &[f64]) -> Result<f64> {
check_same_len(lp1.len(), lp2.len(), "lp1 and lp2")?;
if lp1.is_empty() {
return Err(UcciError::Empty {
what: EMPTY_GENERATION,
});
}
check_logprobs(lp1, "lp1")?;
check_logprobs(lp2, "lp2")?;
let margins: Vec<f64> = lp1
.iter()
.zip(lp2)
.map(|(&x, &y)| {
let (a, b) = (x.exp(), y.exp());
a.max(b).min(1.0) - a.min(b)
})
.collect();
Ok(1.0 - np_mean(&margins))
}
fn check_probabilities(p: &[f64], what: &'static str) -> Result<()> {
match p.iter().position(|&v| !in_prob_range(v)) {
Some(index) => Err(UcciError::OutOfRange {
what,
index,
value: p[index],
expected: "in [0, 1]",
}),
None => Ok(()),
}
}
fn check_logprobs(lp: &[f64], what: &'static str) -> Result<()> {
for (index, &v) in lp.iter().enumerate() {
if v.is_nan() {
return Err(UcciError::NonFinite { what, index });
}
if v > MAX_LOGPROB {
return Err(UcciError::OutOfRange {
what,
index,
value: v,
expected: "a log-probability (<= 0)",
});
}
}
Ok(())
}
pub fn top2_from_candidates(candidates: &[f64]) -> Result<(f64, f64)> {
top2_at(candidates, 0)
}
fn top2_at(candidates: &[f64], token: usize) -> Result<(f64, f64)> {
if candidates.len() < 2 {
return Err(UcciError::TooFewCandidates {
token,
found: candidates.len(),
});
}
let mut first = f64::NEG_INFINITY;
let mut second = f64::NEG_INFINITY;
for &lp in candidates {
if lp.is_nan() {
return Err(UcciError::NonFinite {
what: "log-probabilities",
index: token,
});
}
if lp > first {
second = first;
first = lp;
} else if lp > second {
second = lp;
}
}
if first > MAX_LOGPROB {
return Err(UcciError::OutOfRange {
what: "log-probabilities",
index: token,
value: first,
expected: "a log-probability (<= 0)",
});
}
Ok((first.exp(), second.exp()))
}
pub fn top2_from_logprobs<T: AsRef<[f64]>>(per_token: &[T]) -> Result<Vec<(f64, f64)>> {
per_token
.iter()
.enumerate()
.map(|(t, cands)| top2_at(cands.as_ref(), t))
.collect()
}
pub fn uncertainty_from_top_logprobs<T: AsRef<[f64]>>(per_token: &[T]) -> Result<f64> {
token_margin_uncertainty(&top2_from_logprobs(per_token)?)
}
#[cfg(feature = "json")]
#[cfg_attr(docsrs, doc(cfg(feature = "json")))]
pub fn from_openai_logprobs(content: &serde_json::Value) -> Result<f64> {
let invalid = |reason: String| UcciError::InvalidParameter {
name: "content",
reason,
};
let tokens = content.as_array().ok_or_else(|| {
invalid("expected the array choice.logprobs.content of a chat completion".to_string())
})?;
let mut per_token: Vec<Vec<f64>> = Vec::with_capacity(tokens.len());
for (t, tok) in tokens.iter().enumerate() {
let top = match tok.get("top_logprobs") {
Some(serde_json::Value::Array(top)) => top,
_ => {
return Err(invalid(format!(
"token {t}: top_logprobs is missing; request top_logprobs=2"
)))
}
};
let mut cands = Vec::with_capacity(top.len());
for c in top {
let lp = c
.get("logprob")
.and_then(serde_json::Value::as_f64)
.ok_or_else(|| invalid(format!("token {t}: a candidate has no numeric logprob")))?;
cands.push(lp);
}
per_token.push(cands);
}
uncertainty_from_top_logprobs(&per_token)
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct MarginAccumulator {
margins: Vec<f64>,
}
impl MarginAccumulator {
pub fn new() -> Self {
Self::default()
}
pub fn push_top2(&mut self, a: f64, b: f64) -> Result<f64> {
let m = margin_at(a, b, self.margins.len())?;
self.margins.push(m);
Ok(m)
}
pub fn push_logprobs(&mut self, candidates: &[f64]) -> Result<f64> {
let (p1, p2) = top2_at(candidates, self.margins.len())?;
self.push_top2(p1, p2)
}
pub fn len(&self) -> usize {
self.margins.len()
}
pub fn is_empty(&self) -> bool {
self.margins.is_empty()
}
pub fn margins(&self) -> &[f64] {
&self.margins
}
pub fn clear(&mut self) {
self.margins.clear();
}
pub fn uncertainty(&self) -> Result<f64> {
uncertainty_from_margins(&self.margins)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn close(a: f64, b: f64) -> bool {
(a - b).abs() <= 1e-12
}
#[test]
fn matches_eq4() {
let u = token_margin_uncertainty(&[(0.9, 0.05), (0.6, 0.3), (0.5, 0.5)]).unwrap();
assert!(close(u, 1.0 - (0.85 + 0.3 + 0.0) / 3.0));
}
#[test]
fn max_logprob_constant_is_log1p_of_tolerance() {
assert!((PROBABILITY_TOLERANCE.ln_1p() - MAX_LOGPROB).abs() <= f64::EPSILON * 1e-9);
assert!(MAX_LOGPROB.exp() <= 1.0 + PROBABILITY_TOLERANCE + f64::EPSILON);
}
#[test]
fn pair_order_does_not_matter() {
assert!(close(token_margin_uncertainty(&[(0.1, 0.8)]).unwrap(), 0.3));
assert_eq!(
margins_from_top2(&[(0.1, 0.8), (0.8, 0.1)]).unwrap(),
vec![0.8 - 0.1, 0.8 - 0.1]
);
}
#[test]
fn extremes() {
assert_eq!(token_margin_uncertainty(&[(1.0, 0.0); 4]).unwrap(), 0.0);
assert_eq!(token_margin_uncertainty(&[(0.4, 0.4); 4]).unwrap(), 1.0);
}
#[test]
fn tolerance_above_one_is_capped() {
assert_eq!(token_margin(1.0 + 5e-10, 0.0).unwrap(), 1.0);
assert!(token_margin(1.0 + 1e-8, 0.0).is_err());
assert_eq!(uncertainty_from_probs(&[1.0 + 5e-10], &[0.0]).unwrap(), 0.0);
}
#[test]
fn rejects_bad_probabilities_with_position() {
assert!(token_margin(-0.1, 0.5).is_err());
assert!(token_margin(f64::NAN, 0.5).is_err());
assert!(token_margin(0.5, f64::NAN).is_err());
assert!(token_margin(f64::INFINITY, 0.5).is_err());
match margins_from_top2(&[(0.5, 0.4), (1.5, 0.1)]) {
Err(UcciError::OutOfRange { index, value, .. }) => {
assert_eq!(index, 1);
assert_eq!(value, 1.5);
}
other => panic!("unexpected {other:?}"),
}
match margins_from_top2(&[(0.5, -0.25)]) {
Err(UcciError::OutOfRange { value, .. }) => assert_eq!(value, -0.25),
other => panic!("unexpected {other:?}"),
}
}
#[test]
fn empty_generation() {
assert!(matches!(
token_margin_uncertainty(&[]),
Err(UcciError::Empty { .. })
));
assert!(matches!(
uncertainty_from_margins(&[]),
Err(UcciError::Empty { .. })
));
assert!(uncertainty_from_probs(&[], &[]).is_err());
assert!(uncertainty_from_logprobs(&[], &[]).is_err());
let empty: Vec<Vec<f64>> = vec![];
assert!(uncertainty_from_top_logprobs(&empty).is_err());
assert!(MarginAccumulator::new().uncertainty().is_err());
assert!(MarginAccumulator::new().is_empty());
}
#[test]
fn margins_must_be_valid() {
assert!(uncertainty_from_margins(&[0.5, f64::NAN]).is_err());
assert!(uncertainty_from_margins(&[0.5, 1.5]).is_err());
assert!(uncertainty_from_margins(&[-0.1]).is_err());
assert_eq!(uncertainty_from_margins(&[0.0, 1.0]).unwrap(), 0.5);
}
#[test]
fn array_forms_agree_with_pairs() {
let pairs = [(0.8, 0.1), (0.2, 0.55), (0.99, 0.005), (0.5, 0.5)];
let p1: Vec<f64> = pairs.iter().map(|p| p.0).collect();
let p2: Vec<f64> = pairs.iter().map(|p| p.1).collect();
let a = token_margin_uncertainty(&pairs).unwrap();
assert_eq!(uncertainty_from_probs(&p1, &p2).unwrap(), a);
let l1: Vec<f64> = p1.iter().map(|p| p.ln()).collect();
let l2: Vec<f64> = p2.iter().map(|p| p.ln()).collect();
assert!(close(uncertainty_from_logprobs(&l1, &l2).unwrap(), a));
}
#[test]
fn array_form_errors() {
assert!(matches!(
uncertainty_from_probs(&[0.5], &[0.2, 0.1]),
Err(UcciError::LengthMismatch { .. })
));
assert!(matches!(
uncertainty_from_probs(&[0.5, f64::NAN], &[0.2, 0.1]),
Err(UcciError::OutOfRange { index: 1, .. })
));
assert!(matches!(
uncertainty_from_logprobs(&[-0.1, f64::NAN], &[-2.0, -3.0]),
Err(UcciError::NonFinite { index: 1, .. })
));
assert!(matches!(
uncertainty_from_logprobs(&[0.1], &[-2.0]),
Err(UcciError::OutOfRange { .. })
));
assert!(matches!(
uncertainty_from_logprobs(&[f64::INFINITY], &[-2.0]),
Err(UcciError::OutOfRange { .. })
));
assert_eq!(
uncertainty_from_logprobs(&[0.0], &[f64::NEG_INFINITY]).unwrap(),
0.0
);
}
#[test]
fn top2_from_logprobs_sorts_and_needs_two() {
let out = top2_from_logprobs(&[vec![0.2f64.ln(), 0.7f64.ln(), 0.05f64.ln()]]).unwrap();
assert!(close(out[0].0, 0.7) && close(out[0].1, 0.2));
match top2_from_logprobs(&[vec![0.0, -1.0], vec![0.9f64.ln()]]) {
Err(UcciError::TooFewCandidates { token, found }) => {
assert_eq!((token, found), (1, 1));
}
other => panic!("unexpected {other:?}"),
}
}
#[test]
fn candidate_edge_cases() {
let out = top2_from_candidates(&[-0.5, -0.5, -3.0]).unwrap();
assert_eq!(out.0, out.1);
assert_eq!(
top2_from_candidates(&[f64::NEG_INFINITY, 0.0]).unwrap(),
(1.0, 0.0)
);
assert_eq!(
top2_from_candidates(&[f64::NEG_INFINITY, f64::NEG_INFINITY]).unwrap(),
(0.0, 0.0)
);
assert!(top2_from_candidates(&[f64::NAN, 0.0]).is_err());
assert!(top2_from_candidates(&[-0.1, -0.2, f64::NAN]).is_err());
assert!(top2_from_candidates(&[f64::INFINITY, 0.0]).is_err());
assert!(top2_from_candidates(&[0.5, -1.0]).is_err());
assert!(top2_from_candidates(&[MAX_LOGPROB, -1.0]).is_ok());
assert!(top2_from_candidates(&[]).is_err());
}
#[test]
fn accumulator_matches_batch_and_is_unchanged_by_errors() {
let pairs = [(0.9, 0.1), (0.35, 0.6), (0.7, 0.2)];
let mut acc = MarginAccumulator::new();
for &(a, b) in &pairs {
acc.push_top2(a, b).unwrap();
}
assert!(acc.push_top2(2.0, 0.1).is_err());
assert!(acc.push_logprobs(&[0.0]).is_err());
assert_eq!(acc.len(), 3);
assert_eq!(
acc.uncertainty().unwrap(),
token_margin_uncertainty(&pairs).unwrap()
);
assert_eq!(acc.margins().len(), 3);
acc.clear();
assert!(acc.is_empty());
}
#[test]
fn logprob_paths_agree() {
let probs: [(f64, f64); 3] = [(0.8, 0.1), (0.55, 0.4), (0.99, 0.005)];
let lps: Vec<Vec<f64>> = probs.iter().map(|&(a, b)| vec![b.ln(), a.ln()]).collect();
let a = token_margin_uncertainty(&probs).unwrap();
let b = uncertainty_from_top_logprobs(&lps).unwrap();
let c = token_margin_uncertainty(&top2_from_logprobs(&lps).unwrap()).unwrap();
assert!(close(a, b));
assert_eq!(b, c);
}
#[cfg(feature = "json")]
#[test]
fn openai_content_parsing() {
let content = serde_json::json!([
{"top_logprobs": [{"logprob": 0.8f64.ln()}, {"logprob": 0.1f64.ln()}]},
{"top_logprobs": [{"logprob": 0.55f64.ln()}, {"logprob": 0.4f64.ln()}]}
]);
let u = from_openai_logprobs(&content).unwrap();
assert!(close(u, 1.0 - (0.7 + 0.15) / 2.0));
assert!(from_openai_logprobs(&serde_json::json!({"a": 1})).is_err());
assert!(from_openai_logprobs(&serde_json::json!([{"top_logprobs": null}])).is_err());
assert!(from_openai_logprobs(&serde_json::json!([{"logprob": -0.1}])).is_err());
assert!(from_openai_logprobs(&serde_json::json!([
{"top_logprobs": [{"logprob": "x"}, {"logprob": -1.0}]}
]))
.is_err());
assert!(from_openai_logprobs(&serde_json::json!([])).is_err());
}
}