1use candle_core::Tensor;
4use rand::rngs::StdRng;
5use rand::RngExt;
6
7use crate::{Result, RltError};
8
9#[derive(Clone, Copy, Debug)]
11pub enum Sampler {
12 Greedy,
14 Sample {
17 temperature: f32,
19 top_k: Option<usize>,
21 },
22}
23
24impl Sampler {
25 pub fn behavior_logprob(&self, logits: &Tensor, token: u32) -> Result<f32> {
34 let v = logits.squeeze(0)?.to_vec1::<f32>()?;
35 match self {
36 Sampler::Greedy => {
37 let (idx, _) = argmax(&v)?;
38 Ok(if idx as u32 == token {
39 0.0
40 } else {
41 f32::NEG_INFINITY
42 })
43 }
44 Sampler::Sample { temperature, top_k } => {
45 check_temperature(*temperature)?;
46 let cand = candidates(&v, *temperature, *top_k);
47 if cand.is_empty() {
48 return Err(RltError::State("empty logits".into()));
49 }
50 let Some(j) = cand.iter().position(|&(t, _)| t == token) else {
51 return Ok(f32::NEG_INFINITY);
52 };
53 let logp = log_softmax_vec(&cand.iter().map(|&(_, x)| x).collect::<Vec<_>>());
54 Ok(logp[j])
55 }
56 }
57 }
58}
59
60#[derive(Clone, Debug)]
63pub struct SampledToken {
64 pub token: u32,
66 pub logprob: f32,
71}
72
73pub fn sample_from_logits(
75 logits: &Tensor,
76 sampler: &Sampler,
77 rng: &mut StdRng,
78) -> Result<SampledToken> {
79 let v = logits.squeeze(0)?.to_vec1::<f32>()?;
80 let logp_full = log_softmax_vec(&v);
81 match sampler {
82 Sampler::Greedy => {
83 let (idx, _) = argmax(&v)?;
84 Ok(SampledToken {
85 token: idx as u32,
86 logprob: logp_full[idx],
87 })
88 }
89 Sampler::Sample { temperature, top_k } => {
90 check_temperature(*temperature)?;
91 let cand = candidates(&v, *temperature, *top_k);
92 if cand.is_empty() {
93 return Err(RltError::State("empty logits".into()));
94 }
95 let probs = softmax_vec(&cand.iter().map(|&(_, x)| x).collect::<Vec<_>>());
96 let r = rng.random::<f64>();
97 let mut acc = 0f64;
98 for (j, &(tok, _)) in cand.iter().enumerate() {
99 acc += f64::from(probs[j]);
100 if r <= acc {
101 return Ok(SampledToken {
102 token: tok,
103 logprob: logp_full[tok as usize],
104 });
105 }
106 }
107 let (tok, _) = cand[cand.len() - 1];
108 Ok(SampledToken {
109 token: tok,
110 logprob: logp_full[tok as usize],
111 })
112 }
113 }
114}
115
116fn check_temperature(temperature: f32) -> Result<()> {
117 if !temperature.is_finite() || temperature <= 0.0 {
118 return Err(RltError::Config(
119 "sampling temperature must be finite and > 0".into(),
120 ));
121 }
122 Ok(())
123}
124
125fn argmax(v: &[f32]) -> Result<(usize, f32)> {
126 v.iter()
127 .enumerate()
128 .max_by(|a, b| a.1.total_cmp(b.1))
129 .map(|(i, &x)| (i, x))
130 .ok_or_else(|| RltError::State("empty logits".into()))
131}
132
133fn candidates(v: &[f32], temperature: f32, top_k: Option<usize>) -> Vec<(u32, f32)> {
137 let mut cand: Vec<(u32, f32)> = v
138 .iter()
139 .enumerate()
140 .map(|(i, &x)| (i as u32, x / temperature))
141 .collect();
142 if let Some(k) = top_k {
143 cand.sort_by(|a, b| b.1.total_cmp(&a.1));
144 cand.truncate(k.max(1));
145 }
146 cand
147}
148
149fn softmax_vec(xs: &[f32]) -> Vec<f32> {
150 let m = xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
151 let exps: Vec<f32> = xs.iter().map(|x| (x - m).exp()).collect();
152 let z: f32 = exps.iter().sum();
153 exps.iter().map(|e| e / z).collect()
154}
155
156fn log_softmax_vec(xs: &[f32]) -> Vec<f32> {
157 let m = xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
158 let z: f32 = xs.iter().map(|x| (x - m).exp()).sum();
159 xs.iter().map(|x| x - m - z.ln()).collect()
160}