use candle_core::Tensor;
use rand::rngs::StdRng;
use rand::RngExt;
use crate::{Result, RltError};
#[derive(Clone, Copy, Debug)]
pub enum Sampler {
Greedy,
Sample {
temperature: f32,
top_k: Option<usize>,
},
}
impl Sampler {
pub fn behavior_logprob(&self, logits: &Tensor, token: u32) -> Result<f32> {
let v = logits.squeeze(0)?.to_vec1::<f32>()?;
match self {
Sampler::Greedy => {
let (idx, _) = argmax(&v)?;
Ok(if idx as u32 == token {
0.0
} else {
f32::NEG_INFINITY
})
}
Sampler::Sample { temperature, top_k } => {
check_temperature(*temperature)?;
let cand = candidates(&v, *temperature, *top_k);
if cand.is_empty() {
return Err(RltError::State("empty logits".into()));
}
let Some(j) = cand.iter().position(|&(t, _)| t == token) else {
return Ok(f32::NEG_INFINITY);
};
let logp = log_softmax_vec(&cand.iter().map(|&(_, x)| x).collect::<Vec<_>>());
Ok(logp[j])
}
}
}
}
#[derive(Clone, Debug)]
pub struct SampledToken {
pub token: u32,
pub logprob: f32,
}
pub fn sample_from_logits(
logits: &Tensor,
sampler: &Sampler,
rng: &mut StdRng,
) -> Result<SampledToken> {
let v = logits.squeeze(0)?.to_vec1::<f32>()?;
let logp_full = log_softmax_vec(&v);
match sampler {
Sampler::Greedy => {
let (idx, _) = argmax(&v)?;
Ok(SampledToken {
token: idx as u32,
logprob: logp_full[idx],
})
}
Sampler::Sample { temperature, top_k } => {
check_temperature(*temperature)?;
let cand = candidates(&v, *temperature, *top_k);
if cand.is_empty() {
return Err(RltError::State("empty logits".into()));
}
let probs = softmax_vec(&cand.iter().map(|&(_, x)| x).collect::<Vec<_>>());
let r = rng.random::<f64>();
let mut acc = 0f64;
for (j, &(tok, _)) in cand.iter().enumerate() {
acc += f64::from(probs[j]);
if r <= acc {
return Ok(SampledToken {
token: tok,
logprob: logp_full[tok as usize],
});
}
}
let (tok, _) = cand[cand.len() - 1];
Ok(SampledToken {
token: tok,
logprob: logp_full[tok as usize],
})
}
}
}
fn check_temperature(temperature: f32) -> Result<()> {
if !temperature.is_finite() || temperature <= 0.0 {
return Err(RltError::Config(
"sampling temperature must be finite and > 0".into(),
));
}
Ok(())
}
fn argmax(v: &[f32]) -> Result<(usize, f32)> {
v.iter()
.enumerate()
.max_by(|a, b| a.1.total_cmp(b.1))
.map(|(i, &x)| (i, x))
.ok_or_else(|| RltError::State("empty logits".into()))
}
fn candidates(v: &[f32], temperature: f32, top_k: Option<usize>) -> Vec<(u32, f32)> {
let mut cand: Vec<(u32, f32)> = v
.iter()
.enumerate()
.map(|(i, &x)| (i as u32, x / temperature))
.collect();
if let Some(k) = top_k {
cand.sort_by(|a, b| b.1.total_cmp(&a.1));
cand.truncate(k.max(1));
}
cand
}
fn softmax_vec(xs: &[f32]) -> Vec<f32> {
let m = xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let exps: Vec<f32> = xs.iter().map(|x| (x - m).exp()).collect();
let z: f32 = exps.iter().sum();
exps.iter().map(|e| e / z).collect()
}
fn log_softmax_vec(xs: &[f32]) -> Vec<f32> {
let m = xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let z: f32 = xs.iter().map(|x| (x - m).exp()).sum();
xs.iter().map(|x| x - m - z.ln()).collect()
}