spec_stream_core/
logits.rs1use crate::error::CoreError;
2use crate::types::SamplingParams;
3use rand::Rng;
4use std::collections::HashMap;
5
6pub type TokenId = u32;
8
9#[derive(Debug, Clone, PartialEq)]
11pub struct Logits {
12 pub values: Vec<f32>,
13}
14
15impl Logits {
16 pub fn new(values: Vec<f32>) -> Self {
18 Self { values }
19 }
20
21 #[inline]
23 pub fn vocab_size(&self) -> usize {
24 self.values.len()
25 }
26
27 pub fn argmax(&self) -> Result<TokenId, CoreError> {
29 if self.values.is_empty() {
30 return Err(CoreError::EmptyLogits);
31 }
32
33 let mut max_idx = 0;
34 let mut max_val = self.values[0];
35
36 for (idx, &val) in self.values.iter().enumerate().skip(1) {
37 if val > max_val {
38 max_val = val;
39 max_idx = idx;
40 }
41 }
42
43 Ok(max_idx as TokenId)
44 }
45
46 pub fn apply_temperature(&mut self, temperature: f32) -> Result<(), CoreError> {
48 if temperature <= 0.0 || temperature.is_nan() {
49 return Err(CoreError::InvalidTemperature(temperature));
50 }
51
52 let inv_temp = 1.0 / temperature;
53 for val in &mut self.values {
54 *val *= inv_temp;
55 }
56
57 Ok(())
58 }
59
60 pub fn apply_top_k(&mut self, k: usize) {
63 if k == 0 || k >= self.values.len() {
64 return;
65 }
66
67 let mut sorted_vals: Vec<f32> = self
69 .values
70 .iter()
71 .copied()
72 .filter(|v| !v.is_nan())
73 .collect();
74 sorted_vals.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
75
76 if let Some(&kth_val) = sorted_vals.get(k - 1) {
77 let mut kept = 0;
78 for val in &mut self.values {
79 if *val > kth_val || (*val == kth_val && kept < k) {
80 kept += 1;
81 } else {
82 *val = f32::NEG_INFINITY;
83 }
84 }
85 }
86 }
87
88 pub fn apply_top_p(&mut self, top_p: f32) -> Result<(), CoreError> {
91 if top_p <= 0.0 || top_p > 1.0 || top_p.is_nan() {
92 return Err(CoreError::InvalidTopP(top_p));
93 }
94
95 if top_p >= 1.0 || self.values.len() <= 1 {
96 return Ok(());
97 }
98
99 let probs = self.softmax()?;
100 let mut indexed: Vec<(usize, f32)> = probs.into_iter().enumerate().collect();
101 indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
102
103 let mut cumulative = 0.0f32;
104 let mut cutoff_idx = 0;
105
106 for (rank, &(_, prob)) in indexed.iter().enumerate() {
107 cumulative += prob;
108 cutoff_idx = rank;
109 if cumulative >= top_p {
110 break;
111 }
112 }
113
114 let mut keep = vec![false; self.values.len()];
116 for &(idx, _) in &indexed[0..=cutoff_idx] {
117 keep[idx] = true;
118 }
119
120 for (idx, val) in self.values.iter_mut().enumerate() {
121 if !keep[idx] {
122 *val = f32::NEG_INFINITY;
123 }
124 }
125
126 Ok(())
127 }
128
129 pub fn apply_min_p(&mut self, min_p: f32) -> Result<(), CoreError> {
132 if !(0.0..=1.0).contains(&min_p) || min_p.is_nan() {
133 return Err(CoreError::InvalidMinP(min_p));
134 }
135
136 if min_p <= 0.0 || self.values.is_empty() {
137 return Ok(());
138 }
139
140 let probs = self.softmax()?;
141 let max_prob = probs.iter().copied().fold(0.0f32, f32::max);
142 let threshold = min_p * max_prob;
143
144 for (idx, &prob) in probs.iter().enumerate() {
145 if prob < threshold {
146 self.values[idx] = f32::NEG_INFINITY;
147 }
148 }
149
150 Ok(())
151 }
152
153 pub fn apply_repetition_penalty(
155 &mut self,
156 context_tokens: &[TokenId],
157 penalty: f32,
158 ) -> Result<(), CoreError> {
159 if penalty <= 0.0 || penalty.is_nan() {
160 return Err(CoreError::InvalidPenalty(penalty));
161 }
162
163 if (penalty - 1.0).abs() < 1e-6 || context_tokens.is_empty() {
164 return Ok(());
165 }
166
167 for &token_id in context_tokens {
168 let idx = token_id as usize;
169 if let Some(val) = self.values.get_mut(idx) {
170 if *val > 0.0 {
171 *val /= penalty;
172 } else {
173 *val *= penalty;
174 }
175 }
176 }
177
178 Ok(())
179 }
180
181 pub fn apply_presence_and_frequency_penalties(
183 &mut self,
184 context_tokens: &[TokenId],
185 presence_penalty: f32,
186 frequency_penalty: f32,
187 ) {
188 if (presence_penalty.abs() < 1e-6 && frequency_penalty.abs() < 1e-6)
189 || context_tokens.is_empty()
190 {
191 return;
192 }
193
194 let mut counts = HashMap::new();
195 for &tok in context_tokens {
196 *counts.entry(tok).or_insert(0usize) += 1;
197 }
198
199 for (tok, count) in counts {
200 let idx = tok as usize;
201 if let Some(val) = self.values.get_mut(idx) {
202 *val -= presence_penalty + (count as f32 * frequency_penalty);
203 }
204 }
205 }
206
207 pub fn softmax(&self) -> Result<Vec<f32>, CoreError> {
209 if self.values.is_empty() {
210 return Err(CoreError::EmptyLogits);
211 }
212
213 let max_val = self
214 .values
215 .iter()
216 .copied()
217 .fold(f32::NEG_INFINITY, f32::max);
218
219 let mut exp_sum = 0.0f32;
220 let mut probs = Vec::with_capacity(self.values.len());
221
222 for &v in &self.values {
223 let exp_v = if v == f32::NEG_INFINITY {
224 0.0
225 } else {
226 (v - max_val).exp()
227 };
228 probs.push(exp_v);
229 exp_sum += exp_v;
230 }
231
232 if exp_sum <= 0.0 || exp_sum.is_nan() {
233 let uniform = 1.0 / self.values.len() as f32;
234 return Ok(vec![uniform; self.values.len()]);
235 }
236
237 let inv_sum = 1.0 / exp_sum;
238 for p in &mut probs {
239 *p *= inv_sum;
240 }
241
242 Ok(probs)
243 }
244
245 pub fn sample<R: Rng>(&self, temperature: f32, rng: &mut R) -> Result<TokenId, CoreError> {
247 if temperature < 1e-4 {
248 return self.argmax();
249 }
250
251 let mut scaled = self.clone();
252 scaled.apply_temperature(temperature)?;
253 let probs = scaled.softmax()?;
254
255 let sample_val: f32 = rng.gen();
256 let mut cumulative = 0.0f32;
257
258 for (idx, &p) in probs.iter().enumerate() {
259 cumulative += p;
260 if sample_val <= cumulative {
261 return Ok(idx as TokenId);
262 }
263 }
264
265 Ok((self.values.len() - 1) as TokenId)
267 }
268
269 pub fn sample_with_params<R: Rng>(
271 &self,
272 params: &SamplingParams,
273 context_tokens: &[TokenId],
274 rng: &mut R,
275 ) -> Result<TokenId, CoreError> {
276 if params.temperature < 1e-4 {
277 let mut adjusted = self.clone();
278 adjusted.apply_repetition_penalty(context_tokens, params.repetition_penalty)?;
279 adjusted.apply_presence_and_frequency_penalties(
280 context_tokens,
281 params.presence_penalty,
282 params.frequency_penalty,
283 );
284 return adjusted.argmax();
285 }
286
287 let mut processed = self.clone();
288 processed.apply_repetition_penalty(context_tokens, params.repetition_penalty)?;
289 processed.apply_presence_and_frequency_penalties(
290 context_tokens,
291 params.presence_penalty,
292 params.frequency_penalty,
293 );
294 processed.apply_temperature(params.temperature)?;
295
296 if params.top_k > 0 {
297 processed.apply_top_k(params.top_k);
298 }
299
300 if params.top_p < 1.0 {
301 processed.apply_top_p(params.top_p)?;
302 }
303
304 if params.min_p > 0.0 {
305 processed.apply_min_p(params.min_p)?;
306 }
307
308 let probs = processed.softmax()?;
309 let sample_val: f32 = rng.gen();
310 let mut cumulative = 0.0f32;
311
312 for (idx, &p) in probs.iter().enumerate() {
313 cumulative += p;
314 if sample_val <= cumulative {
315 return Ok(idx as TokenId);
316 }
317 }
318
319 Ok((self.values.len() - 1) as TokenId)
320 }
321}
322
323#[cfg(test)]
324mod tests {
325 use super::*;
326 use rand::rngs::StdRng;
327 use rand::SeedableRng;
328
329 #[test]
330 fn test_top_k_filtering() {
331 let mut logits = Logits::new(vec![1.0, 10.0, 5.0, 2.0, 8.0]);
332 logits.apply_top_k(2); assert_eq!(logits.values[1], 10.0);
334 assert_eq!(logits.values[4], 8.0);
335 assert_eq!(logits.values[0], f32::NEG_INFINITY);
336 assert_eq!(logits.values[2], f32::NEG_INFINITY);
337 assert_eq!(logits.values[3], f32::NEG_INFINITY);
338 }
339
340 #[test]
341 fn test_top_p_filtering() {
342 let mut logits = Logits::new(vec![10.0, 1.0, 0.5, 0.1]);
343 logits.apply_top_p(0.8).unwrap();
344 assert_eq!(logits.values[0], 10.0);
346 assert_eq!(logits.values[1], f32::NEG_INFINITY);
347 }
348
349 #[test]
350 fn test_repetition_penalty() {
351 let mut logits = Logits::new(vec![4.0, -2.0, 5.0]);
352 logits.apply_repetition_penalty(&[0, 1], 2.0).unwrap();
353 assert!((logits.values[0] - 2.0).abs() < 1e-5); assert!((logits.values[1] - (-4.0)).abs() < 1e-5); assert_eq!(logits.values[2], 5.0); }
357
358 #[test]
359 fn test_sampling_with_params() {
360 let logits = Logits::new(vec![1.0, 10.0, 2.0]);
361 let mut rng = StdRng::seed_from_u64(42);
362 let params = SamplingParams {
363 temperature: 0.1,
364 top_k: 1,
365 ..Default::default()
366 };
367 let token = logits.sample_with_params(¶ms, &[], &mut rng).unwrap();
368 assert_eq!(token, 1);
369 }
370}