use alloc::sync::Arc;
use core::fmt::Debug;
use burn::{
Tensor,
config::Config,
};
use super::{
BeamSearch,
GreedySearch,
LogitFilter,
MaximumLikelihoodRanker,
MinP,
TopK,
TopP,
};
pub trait TokenSearch: Send + Debug {
fn group_size(&self) -> usize;
fn reset(&mut self);
fn update(
&mut self,
tokens: &mut Vec<Vec<i64>>,
logits: Tensor<2>,
sum_logprobs: &mut [f32],
reorder: &mut dyn FnMut(&[usize]),
) -> (Vec<i64>, bool);
fn finalize(
&mut self,
tokens: Vec<Vec<i64>>,
sum_logprobs: Vec<f32>,
prompt_len: usize,
) -> Vec<Vec<Candidate>>;
}
#[derive(Clone, Debug, PartialEq)]
pub struct Candidate {
pub tokens: Vec<i64>,
pub sum_logprob: f32,
pub stop: Option<i64>,
}
impl Candidate {
pub fn tokens_with_stop(&self) -> Vec<i64> {
let mut out = self.tokens.clone();
out.extend(self.stop);
out
}
}
#[derive(Config, Debug, PartialEq)]
pub struct SearchConfig {
#[config(default = "1")]
pub beam_size: usize,
#[config(default = "None")]
pub patience: Option<f64>,
#[config(default = "None")]
pub length_penalty: Option<f64>,
#[config(default = "0.0")]
pub temperature: f64,
#[config(default = "None")]
pub best_of: Option<usize>,
#[config(default = "None")]
pub top_k: Option<usize>,
#[config(default = "None")]
pub top_p: Option<f64>,
#[config(default = "None")]
pub min_p: Option<f64>,
}
impl SearchConfig {
pub fn init_search(
&self,
stops: &[i64],
filler: i64,
) -> Box<dyn TokenSearch> {
let stops = stops.iter().copied();
if self.temperature > 0.0 {
Box::new(
GreedySearch::new(stops, filler)
.with_temperature(self.temperature)
.with_group(self.best_of.unwrap_or(1)),
)
} else if self.beam_size == 1 {
Box::new(GreedySearch::new(stops, filler))
} else {
Box::new(BeamSearch::new(self.beam_size, stops, self.patience))
}
}
pub fn sampling_filters(&self) -> Vec<Arc<dyn LogitFilter>> {
let mut filters: Vec<Arc<dyn LogitFilter>> = Vec::new();
if self.temperature <= 0.0 {
return filters;
}
let temperature = self.temperature;
if let Some(k) = self.top_k {
filters.push(Arc::new(TopK { k }));
}
if let Some(p) = self.top_p {
filters.push(Arc::new(TopP { p, temperature }));
}
if let Some(p) = self.min_p {
filters.push(Arc::new(MinP { p, temperature }));
}
filters
}
pub fn init_ranker(&self) -> MaximumLikelihoodRanker {
MaximumLikelihoodRanker {
length_penalty: self.length_penalty,
}
}
pub fn at_temperature(
&self,
temperature: f64,
best_of: Option<usize>,
) -> Self {
let mut config = self.clone().with_temperature(temperature);
if temperature > 0.0 {
config.beam_size = 1;
config.patience = None;
config.best_of = best_of;
} else {
config.best_of = None;
}
config
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_init_search_and_rungs() {
let base = SearchConfig::new()
.with_beam_size(5)
.with_patience(Some(2.0))
.with_length_penalty(Some(0.6));
assert_eq!(base.init_search(&[9], 0).group_size(), 5);
assert_eq!(base.init_ranker().length_penalty, Some(0.6));
let sampled = base.at_temperature(0.4, Some(3));
assert_eq!(sampled.beam_size, 1);
assert_eq!(sampled.patience, None);
assert_eq!(sampled.best_of, Some(3));
assert_eq!(sampled.init_search(&[9], 0).group_size(), 3);
let searched = base.at_temperature(0.0, Some(3));
assert_eq!(searched.beam_size, 5);
assert_eq!(searched.best_of, None);
assert_eq!(SearchConfig::new().init_search(&[9], 0).group_size(), 1);
}
#[test]
fn test_sampling_filters() {
let warm = SearchConfig::new()
.with_temperature(0.6)
.with_top_k(Some(20))
.with_top_p(Some(0.95));
assert_eq!(warm.sampling_filters().len(), 2);
assert!(
warm.clone()
.with_temperature(0.0)
.sampling_filters()
.is_empty()
);
assert!(SearchConfig::new().sampling_filters().is_empty());
}
}