Skip to main content

spec_stream_core/
logits.rs

1use crate::error::CoreError;
2use crate::types::SamplingParams;
3use rand::Rng;
4use std::collections::HashMap;
5
6/// Strongly-typed token ID representation.
7pub type TokenId = u32;
8
9/// A vector of raw unnormalized model logits.
10#[derive(Debug, Clone, PartialEq)]
11pub struct Logits {
12    pub values: Vec<f32>,
13}
14
15impl Logits {
16    /// Creates a new Logits instance from a raw vector of floats.
17    pub fn new(values: Vec<f32>) -> Self {
18        Self { values }
19    }
20
21    /// Number of vocabulary items.
22    #[inline]
23    pub fn vocab_size(&self) -> usize {
24        self.values.len()
25    }
26
27    /// Computes the greedy argmax token index.
28    pub fn argmax(&self) -> Result<TokenId, CoreError> {
29        if self.values.is_empty() {
30            return Err(CoreError::EmptyLogits);
31        }
32
33        let mut max_idx = 0;
34        let mut max_val = self.values[0];
35
36        for (idx, &val) in self.values.iter().enumerate().skip(1) {
37            if val > max_val {
38                max_val = val;
39                max_idx = idx;
40            }
41        }
42
43        Ok(max_idx as TokenId)
44    }
45
46    /// Scales the logits by a positive temperature parameter.
47    pub fn apply_temperature(&mut self, temperature: f32) -> Result<(), CoreError> {
48        if temperature <= 0.0 || temperature.is_nan() {
49            return Err(CoreError::InvalidTemperature(temperature));
50        }
51
52        let inv_temp = 1.0 / temperature;
53        for val in &mut self.values {
54            *val *= inv_temp;
55        }
56
57        Ok(())
58    }
59
60    /// Filters logits to keep only the top-K highest scoring tokens.
61    /// All other positions are set to -INFINITY.
62    pub fn apply_top_k(&mut self, k: usize) {
63        if k == 0 || k >= self.values.len() {
64            return;
65        }
66
67        // Find the k-th largest value by sorting indexed pairs or partitioning
68        let mut sorted_vals: Vec<f32> = self
69            .values
70            .iter()
71            .copied()
72            .filter(|v| !v.is_nan())
73            .collect();
74        sorted_vals.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
75
76        if let Some(&kth_val) = sorted_vals.get(k - 1) {
77            let mut kept = 0;
78            for val in &mut self.values {
79                if *val > kth_val || (*val == kth_val && kept < k) {
80                    kept += 1;
81                } else {
82                    *val = f32::NEG_INFINITY;
83                }
84            }
85        }
86    }
87
88    /// Filters logits using nucleus (top-p) sampling.
89    /// Keeps the smallest set of tokens whose cumulative probability is >= top_p.
90    pub fn apply_top_p(&mut self, top_p: f32) -> Result<(), CoreError> {
91        if top_p <= 0.0 || top_p > 1.0 || top_p.is_nan() {
92            return Err(CoreError::InvalidTopP(top_p));
93        }
94
95        if top_p >= 1.0 || self.values.len() <= 1 {
96            return Ok(());
97        }
98
99        let probs = self.softmax()?;
100        let mut indexed: Vec<(usize, f32)> = probs.into_iter().enumerate().collect();
101        indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
102
103        let mut cumulative = 0.0f32;
104        let mut cutoff_idx = 0;
105
106        for (rank, &(_, prob)) in indexed.iter().enumerate() {
107            cumulative += prob;
108            cutoff_idx = rank;
109            if cumulative >= top_p {
110                break;
111            }
112        }
113
114        // Indices to keep: indexed[0..=cutoff_idx]
115        let mut keep = vec![false; self.values.len()];
116        for &(idx, _) in &indexed[0..=cutoff_idx] {
117            keep[idx] = true;
118        }
119
120        for (idx, val) in self.values.iter_mut().enumerate() {
121            if !keep[idx] {
122                *val = f32::NEG_INFINITY;
123            }
124        }
125
126        Ok(())
127    }
128
129    /// Filters logits using min-p sampling.
130    /// Retains tokens with probability >= min_p * max_probability.
131    pub fn apply_min_p(&mut self, min_p: f32) -> Result<(), CoreError> {
132        if !(0.0..=1.0).contains(&min_p) || min_p.is_nan() {
133            return Err(CoreError::InvalidMinP(min_p));
134        }
135
136        if min_p <= 0.0 || self.values.is_empty() {
137            return Ok(());
138        }
139
140        let probs = self.softmax()?;
141        let max_prob = probs.iter().copied().fold(0.0f32, f32::max);
142        let threshold = min_p * max_prob;
143
144        for (idx, &prob) in probs.iter().enumerate() {
145            if prob < threshold {
146                self.values[idx] = f32::NEG_INFINITY;
147            }
148        }
149
150        Ok(())
151    }
152
153    /// Applies repetition penalty to tokens that have already been generated in the context.
154    pub fn apply_repetition_penalty(
155        &mut self,
156        context_tokens: &[TokenId],
157        penalty: f32,
158    ) -> Result<(), CoreError> {
159        if penalty <= 0.0 || penalty.is_nan() {
160            return Err(CoreError::InvalidPenalty(penalty));
161        }
162
163        if (penalty - 1.0).abs() < 1e-6 || context_tokens.is_empty() {
164            return Ok(());
165        }
166
167        for &token_id in context_tokens {
168            let idx = token_id as usize;
169            if let Some(val) = self.values.get_mut(idx) {
170                if *val > 0.0 {
171                    *val /= penalty;
172                } else {
173                    *val *= penalty;
174                }
175            }
176        }
177
178        Ok(())
179    }
180
181    /// Applies presence and frequency penalties (OpenAI style).
182    pub fn apply_presence_and_frequency_penalties(
183        &mut self,
184        context_tokens: &[TokenId],
185        presence_penalty: f32,
186        frequency_penalty: f32,
187    ) {
188        if (presence_penalty.abs() < 1e-6 && frequency_penalty.abs() < 1e-6)
189            || context_tokens.is_empty()
190        {
191            return;
192        }
193
194        let mut counts = HashMap::new();
195        for &tok in context_tokens {
196            *counts.entry(tok).or_insert(0usize) += 1;
197        }
198
199        for (tok, count) in counts {
200            let idx = tok as usize;
201            if let Some(val) = self.values.get_mut(idx) {
202                *val -= presence_penalty + (count as f32 * frequency_penalty);
203            }
204        }
205    }
206
207    /// Computes numerically stable softmax probabilities.
208    pub fn softmax(&self) -> Result<Vec<f32>, CoreError> {
209        if self.values.is_empty() {
210            return Err(CoreError::EmptyLogits);
211        }
212
213        let max_val = self
214            .values
215            .iter()
216            .copied()
217            .fold(f32::NEG_INFINITY, f32::max);
218
219        let mut exp_sum = 0.0f32;
220        let mut probs = Vec::with_capacity(self.values.len());
221
222        for &v in &self.values {
223            let exp_v = if v == f32::NEG_INFINITY {
224                0.0
225            } else {
226                (v - max_val).exp()
227            };
228            probs.push(exp_v);
229            exp_sum += exp_v;
230        }
231
232        if exp_sum <= 0.0 || exp_sum.is_nan() {
233            let uniform = 1.0 / self.values.len() as f32;
234            return Ok(vec![uniform; self.values.len()]);
235        }
236
237        let inv_sum = 1.0 / exp_sum;
238        for p in &mut probs {
239            *p *= inv_sum;
240        }
241
242        Ok(probs)
243    }
244
245    /// Samples a token from the distribution using standard inverse transform sampling.
246    pub fn sample<R: Rng>(&self, temperature: f32, rng: &mut R) -> Result<TokenId, CoreError> {
247        if temperature < 1e-4 {
248            return self.argmax();
249        }
250
251        let mut scaled = self.clone();
252        scaled.apply_temperature(temperature)?;
253        let probs = scaled.softmax()?;
254
255        let sample_val: f32 = rng.gen();
256        let mut cumulative = 0.0f32;
257
258        for (idx, &p) in probs.iter().enumerate() {
259            cumulative += p;
260            if sample_val <= cumulative {
261                return Ok(idx as TokenId);
262            }
263        }
264
265        // Fallback in case of floating-point rounding
266        Ok((self.values.len() - 1) as TokenId)
267    }
268
269    /// Samples a token using full sampling parameters (temperature, top_k, top_p, penalties).
270    pub fn sample_with_params<R: Rng>(
271        &self,
272        params: &SamplingParams,
273        context_tokens: &[TokenId],
274        rng: &mut R,
275    ) -> Result<TokenId, CoreError> {
276        if params.temperature < 1e-4 {
277            let mut adjusted = self.clone();
278            adjusted.apply_repetition_penalty(context_tokens, params.repetition_penalty)?;
279            adjusted.apply_presence_and_frequency_penalties(
280                context_tokens,
281                params.presence_penalty,
282                params.frequency_penalty,
283            );
284            return adjusted.argmax();
285        }
286
287        let mut processed = self.clone();
288        processed.apply_repetition_penalty(context_tokens, params.repetition_penalty)?;
289        processed.apply_presence_and_frequency_penalties(
290            context_tokens,
291            params.presence_penalty,
292            params.frequency_penalty,
293        );
294        processed.apply_temperature(params.temperature)?;
295
296        if params.top_k > 0 {
297            processed.apply_top_k(params.top_k);
298        }
299
300        if params.top_p < 1.0 {
301            processed.apply_top_p(params.top_p)?;
302        }
303
304        if params.min_p > 0.0 {
305            processed.apply_min_p(params.min_p)?;
306        }
307
308        let probs = processed.softmax()?;
309        let sample_val: f32 = rng.gen();
310        let mut cumulative = 0.0f32;
311
312        for (idx, &p) in probs.iter().enumerate() {
313            cumulative += p;
314            if sample_val <= cumulative {
315                return Ok(idx as TokenId);
316            }
317        }
318
319        Ok((self.values.len() - 1) as TokenId)
320    }
321}
322
323#[cfg(test)]
324mod tests {
325    use super::*;
326    use rand::rngs::StdRng;
327    use rand::SeedableRng;
328
329    #[test]
330    fn test_top_k_filtering() {
331        let mut logits = Logits::new(vec![1.0, 10.0, 5.0, 2.0, 8.0]);
332        logits.apply_top_k(2); // Should keep 10.0 (idx 1) and 8.0 (idx 4)
333        assert_eq!(logits.values[1], 10.0);
334        assert_eq!(logits.values[4], 8.0);
335        assert_eq!(logits.values[0], f32::NEG_INFINITY);
336        assert_eq!(logits.values[2], f32::NEG_INFINITY);
337        assert_eq!(logits.values[3], f32::NEG_INFINITY);
338    }
339
340    #[test]
341    fn test_top_p_filtering() {
342        let mut logits = Logits::new(vec![10.0, 1.0, 0.5, 0.1]);
343        logits.apply_top_p(0.8).unwrap();
344        // Index 0 has vastly dominant probability (~0.999), so only it is kept
345        assert_eq!(logits.values[0], 10.0);
346        assert_eq!(logits.values[1], f32::NEG_INFINITY);
347    }
348
349    #[test]
350    fn test_repetition_penalty() {
351        let mut logits = Logits::new(vec![4.0, -2.0, 5.0]);
352        logits.apply_repetition_penalty(&[0, 1], 2.0).unwrap();
353        assert!((logits.values[0] - 2.0).abs() < 1e-5); // 4.0 / 2.0
354        assert!((logits.values[1] - (-4.0)).abs() < 1e-5); // -2.0 * 2.0
355        assert_eq!(logits.values[2], 5.0); // unchanged
356    }
357
358    #[test]
359    fn test_sampling_with_params() {
360        let logits = Logits::new(vec![1.0, 10.0, 2.0]);
361        let mut rng = StdRng::seed_from_u64(42);
362        let params = SamplingParams {
363            temperature: 0.1,
364            top_k: 1,
365            ..Default::default()
366        };
367        let token = logits.sample_with_params(&params, &[], &mut rng).unwrap();
368        assert_eq!(token, 1);
369    }
370}