use burn::{
Tensor,
prelude::Int,
tensor::activation::softmax,
};
use super::LogitFilter;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TopK {
pub k: usize,
}
impl LogitFilter for TopK {
fn apply(
&self,
logits: Tensor<2>,
_tokens: &[Vec<i64>],
_prompt_len: usize,
) -> Tensor<2> {
let [rows, vocab] = logits.dims();
let k = self.k.clamp(1, vocab);
let kth = logits.clone().topk(k, 1).slice_dim(1, k - 1..k);
let below = logits.clone().lower(kth.expand([rows, vocab]));
logits.mask_fill(below, f32::NEG_INFINITY)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct TopP {
pub p: f64,
pub temperature: f64,
}
impl LogitFilter for TopP {
fn apply(
&self,
logits: Tensor<2>,
_tokens: &[Vec<i64>],
_prompt_len: usize,
) -> Tensor<2> {
let [rows, vocab] = logits.dims();
let sorted = logits.clone().sort_descending(1);
let probs = softmax(sorted.clone() / self.temperature.max(f64::MIN_POSITIVE), 1);
let before = probs.clone().cumsum(1) - probs;
let kept: Tensor<2, Int> = before.lower_elem(self.p).int().sum_dim(1).clamp_min(1);
let threshold = sorted.gather(1, kept - 1);
let below = logits.clone().lower(threshold.expand([rows, vocab]));
logits.mask_fill(below, f32::NEG_INFINITY)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct MinP {
pub p: f64,
pub temperature: f64,
}
impl LogitFilter for MinP {
fn apply(
&self,
logits: Tensor<2>,
_tokens: &[Vec<i64>],
_prompt_len: usize,
) -> Tensor<2> {
let [rows, vocab] = logits.dims();
let floor = logits.clone().max_dim(1) + self.temperature * self.p.ln();
let below = logits.clone().lower(floor.expand([rows, vocab]));
logits.mask_fill(below, f32::NEG_INFINITY)
}
}
#[cfg(test)]
mod tests {
use burn::prelude::TensorData;
use super::*;
use crate::support::testing::cpu_device;
fn kept(
filter: &dyn LogitFilter,
row: &[f32],
) -> Vec<usize> {
let logits =
Tensor::<2>::from_data(TensorData::new(row.to_vec(), [1, row.len()]), &cpu_device());
let out: Vec<f32> = filter
.apply(logits, &[], 0)
.into_data()
.try_into_vec_as::<f32>()
.unwrap();
(0..row.len()).filter(|&i| out[i].is_finite()).collect()
}
#[test]
fn test_top_k() {
let row = [1.0, 4.0, 3.0, 2.0, 0.0];
assert_eq!(kept(&TopK { k: 2 }, &row), [1, 2]);
assert_eq!(kept(&TopK { k: 9 }, &row), [0, 1, 2, 3, 4]);
}
#[test]
fn test_top_p() {
let row = [0.5f32.ln(), 0.3f32.ln(), 0.15f32.ln(), 0.05f32.ln()];
let at = |p| TopP {
p,
temperature: 1.0,
};
assert_eq!(kept(&at(0.5), &row), [0]);
assert_eq!(kept(&at(0.6), &row), [0, 1]);
assert_eq!(kept(&at(0.9), &row), [0, 1, 2]);
assert_eq!(kept(&at(1.0), &row), [0, 1, 2, 3]);
assert_eq!(kept(&at(0.0001), &row), [0], "the top token always");
assert_eq!(
kept(
&TopP {
p: 0.6,
temperature: 0.25
},
&row
),
[0]
);
}
#[test]
fn test_min_p() {
let row = [0.5f32.ln(), 0.3f32.ln(), 0.15f32.ln(), 0.05f32.ln()];
assert_eq!(
kept(
&MinP {
p: 0.5,
temperature: 1.0
},
&row
),
[0, 1]
);
assert_eq!(
kept(
&MinP {
p: 0.2,
temperature: 1.0
},
&row
),
[0, 1, 2]
);
assert_eq!(
kept(
&MinP {
p: 0.5,
temperature: 0.5
},
&row
),
[0]
);
}
}