Skip to main content

rlt_core/
sampling.rs

1//! Sampling from readout logits: greedy, temperature, top-k.
2
3use candle_core::Tensor;
4use rand::rngs::StdRng;
5use rand::RngExt;
6
7use crate::{Result, RltError};
8
9/// Sampling strategy for one token.
10#[derive(Clone, Copy, Debug)]
11pub enum Sampler {
12    /// Deterministic argmax.
13    Greedy,
14    /// Stochastic sampling with temperature and optional top-k truncation
15    /// (the truncated distribution is renormalized, matching replay requirements).
16    Sample {
17        /// Temperature (finite, > 0); divides logits before softmax.
18        temperature: f32,
19        /// Optional top-k truncation.
20        top_k: Option<usize>,
21    },
22}
23
24impl Sampler {
25    /// log-probability of `token` under this sampler's ACTUAL sampling distribution
26    /// (paper §5.3: behavior log-probabilities must describe the distribution that
27    /// produced the action, including temperature and truncation/renormalization).
28    ///
29    /// - `Greedy`: point mass at the argmax — 0.0 for the argmax, −∞ otherwise.
30    /// - `Sample`: logits scaled by 1/temperature, optionally top-k truncated; a
31    ///   token outside the candidate set has probability 0 (−∞ log-prob). For
32    ///   temperature 1.0 without top-k this equals the model's full-vocab log-prob.
33    pub fn behavior_logprob(&self, logits: &Tensor, token: u32) -> Result<f32> {
34        let v = logits.squeeze(0)?.to_vec1::<f32>()?;
35        match self {
36            Sampler::Greedy => {
37                let (idx, _) = argmax(&v)?;
38                Ok(if idx as u32 == token {
39                    0.0
40                } else {
41                    f32::NEG_INFINITY
42                })
43            }
44            Sampler::Sample { temperature, top_k } => {
45                check_temperature(*temperature)?;
46                let cand = candidates(&v, *temperature, *top_k);
47                if cand.is_empty() {
48                    return Err(RltError::State("empty logits".into()));
49                }
50                let Some(j) = cand.iter().position(|&(t, _)| t == token) else {
51                    return Ok(f32::NEG_INFINITY);
52                };
53                let logp = log_softmax_vec(&cand.iter().map(|&(_, x)| x).collect::<Vec<_>>());
54                Ok(logp[j])
55            }
56        }
57    }
58}
59
60/// A sampled token and its log-probability under the model's full-vocab
61/// distribution.
62#[derive(Clone, Debug)]
63pub struct SampledToken {
64    /// Sampled token id.
65    pub token: u32,
66    /// The model's full-vocab log-probability of this token. Valid as the RL
67    /// behavior log-probability ONLY for untruncated sampling at temperature 1.0;
68    /// under any other sampler use [`Sampler::behavior_logprob`], which describes
69    /// the actual sampling distribution (paper §5.3).
70    pub logprob: f32,
71}
72
73/// Sample one token from logits (1, vocab).
74pub fn sample_from_logits(
75    logits: &Tensor,
76    sampler: &Sampler,
77    rng: &mut StdRng,
78) -> Result<SampledToken> {
79    let v = logits.squeeze(0)?.to_vec1::<f32>()?;
80    let logp_full = log_softmax_vec(&v);
81    match sampler {
82        Sampler::Greedy => {
83            let (idx, _) = argmax(&v)?;
84            Ok(SampledToken {
85                token: idx as u32,
86                logprob: logp_full[idx],
87            })
88        }
89        Sampler::Sample { temperature, top_k } => {
90            check_temperature(*temperature)?;
91            let cand = candidates(&v, *temperature, *top_k);
92            if cand.is_empty() {
93                return Err(RltError::State("empty logits".into()));
94            }
95            let probs = softmax_vec(&cand.iter().map(|&(_, x)| x).collect::<Vec<_>>());
96            let r = rng.random::<f64>();
97            let mut acc = 0f64;
98            for (j, &(tok, _)) in cand.iter().enumerate() {
99                acc += f64::from(probs[j]);
100                if r <= acc {
101                    return Ok(SampledToken {
102                        token: tok,
103                        logprob: logp_full[tok as usize],
104                    });
105                }
106            }
107            let (tok, _) = cand[cand.len() - 1];
108            Ok(SampledToken {
109                token: tok,
110                logprob: logp_full[tok as usize],
111            })
112        }
113    }
114}
115
116fn check_temperature(temperature: f32) -> Result<()> {
117    if !temperature.is_finite() || temperature <= 0.0 {
118        return Err(RltError::Config(
119            "sampling temperature must be finite and > 0".into(),
120        ));
121    }
122    Ok(())
123}
124
125fn argmax(v: &[f32]) -> Result<(usize, f32)> {
126    v.iter()
127        .enumerate()
128        .max_by(|a, b| a.1.total_cmp(b.1))
129        .map(|(i, &x)| (i, x))
130        .ok_or_else(|| RltError::State("empty logits".into()))
131}
132
133/// Candidate list for `Sample`: logits scaled by 1/temperature, optionally
134/// truncated to the top-k highest (k clamped to >= 1). Renormalization is the
135/// caller's job (`softmax_vec` / `log_softmax_vec`).
136fn candidates(v: &[f32], temperature: f32, top_k: Option<usize>) -> Vec<(u32, f32)> {
137    let mut cand: Vec<(u32, f32)> = v
138        .iter()
139        .enumerate()
140        .map(|(i, &x)| (i as u32, x / temperature))
141        .collect();
142    if let Some(k) = top_k {
143        cand.sort_by(|a, b| b.1.total_cmp(&a.1));
144        cand.truncate(k.max(1));
145    }
146    cand
147}
148
149fn softmax_vec(xs: &[f32]) -> Vec<f32> {
150    let m = xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
151    let exps: Vec<f32> = xs.iter().map(|x| (x - m).exp()).collect();
152    let z: f32 = exps.iter().sum();
153    exps.iter().map(|e| e / z).collect()
154}
155
156fn log_softmax_vec(xs: &[f32]) -> Vec<f32> {
157    let m = xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
158    let z: f32 = xs.iter().map(|x| (x - m).exp()).sum();
159    xs.iter().map(|x| x - m - z.ln()).collect()
160}