pub struct Sampler { /* private fields */ }Expand description
How the next token is drawn from a row of logits.
Built by Sampler::greedy or Sampler::temperature and narrowed with
the chained methods:
let mut sampler = Sampler::temperature(0.8, Some(42))
.top_k(40)
.top_p(0.95)
.repetition_penalty(1.1);
let next = sampler.pick(&[0.1, 3.0, 0.2], &[]);Implementations§
Source§impl Sampler
impl Sampler
Sourcepub fn greedy() -> Self
pub fn greedy() -> Self
Always takes the highest-scoring token. Reproducible by construction, and the right default for anything being measured rather than read.
Sourcepub fn temperature(temperature: f32, seed: Option<u64>) -> Self
pub fn temperature(temperature: f32, seed: Option<u64>) -> Self
Draws from the distribution after dividing the logits by temperature:
below one sharpens it, above one flattens it, and zero is
Sampler::greedy.
seed of None takes one from the operating system, so two runs of the
same prompt differ.
Examples found in repository?
216fn generate(prompt: &str, new_tokens: usize) -> Result<(), Box<dyn std::error::Error>> {
217 let tokenizer = CharTokenizer::load(VOCABULARY)
218 .map_err(|_| format!("{VOCABULARY} not found - run `language_model -- train` first"))?;
219 let model = TransformerLm::load_bin(WEIGHTS)?;
220
221 let ids = tokenizer.encode(prompt);
222 if ids.is_empty() {
223 return Err("the prompt has no characters this model was trained on".into());
224 }
225
226 // Two temperatures from the same prompt, because the difference is the
227 // whole point: low is repetitive and safe, high is varied and wrong more
228 // often. Top-k matters as much as the temperature: without it the thirty
229 // thousand individually-impossible tokens carry enough probability between
230 // them that one is eventually drawn, and a single wrong token derails
231 // everything after it.
232 for (temperature, top_k) in [(0.2, 8), (0.8, 8)] {
233 // A fixed seed, so re-running the example on the same checkpoint prints
234 // the same text; pass `None` for a different sample each run.
235 let mut sampler = Sampler::temperature(temperature, Some(42)).top_k(top_k);
236
237 // Printed as it decodes rather than at the end: each token costs a
238 // forward pass, and on a real model that is several seconds of nothing.
239 // `flush`, because stdout is line buffered and none of this is a line.
240 print!("T={temperature:.1} k={top_k}\n {}", tokenizer.decode(&ids));
241 model.generate_with(&ids, new_tokens, &mut sampler, |id| {
242 print!("{}", tokenizer.decode(&[id]));
243 let _ = std::io::Write::flush(&mut std::io::stdout());
244 true
245 })?;
246 println!("\n");
247 }
248
249 Ok(())
250}Sourcepub fn top_k(self, k: usize) -> Self
pub fn top_k(self, k: usize) -> Self
Keeps only the k highest-scoring tokens. Zero keeps all of them.
Examples found in repository?
216fn generate(prompt: &str, new_tokens: usize) -> Result<(), Box<dyn std::error::Error>> {
217 let tokenizer = CharTokenizer::load(VOCABULARY)
218 .map_err(|_| format!("{VOCABULARY} not found - run `language_model -- train` first"))?;
219 let model = TransformerLm::load_bin(WEIGHTS)?;
220
221 let ids = tokenizer.encode(prompt);
222 if ids.is_empty() {
223 return Err("the prompt has no characters this model was trained on".into());
224 }
225
226 // Two temperatures from the same prompt, because the difference is the
227 // whole point: low is repetitive and safe, high is varied and wrong more
228 // often. Top-k matters as much as the temperature: without it the thirty
229 // thousand individually-impossible tokens carry enough probability between
230 // them that one is eventually drawn, and a single wrong token derails
231 // everything after it.
232 for (temperature, top_k) in [(0.2, 8), (0.8, 8)] {
233 // A fixed seed, so re-running the example on the same checkpoint prints
234 // the same text; pass `None` for a different sample each run.
235 let mut sampler = Sampler::temperature(temperature, Some(42)).top_k(top_k);
236
237 // Printed as it decodes rather than at the end: each token costs a
238 // forward pass, and on a real model that is several seconds of nothing.
239 // `flush`, because stdout is line buffered and none of this is a line.
240 print!("T={temperature:.1} k={top_k}\n {}", tokenizer.decode(&ids));
241 model.generate_with(&ids, new_tokens, &mut sampler, |id| {
242 print!("{}", tokenizer.decode(&[id]));
243 let _ = std::io::Write::flush(&mut std::io::stdout());
244 true
245 })?;
246 println!("\n");
247 }
248
249 Ok(())
250}Sourcepub fn top_p(self, p: f32) -> Self
pub fn top_p(self, p: f32) -> Self
Keeps the smallest set of tokens whose probabilities reach p, so the
number kept follows how confident the model is. One keeps all of them.
Sourcepub fn min_p(self, p: f32) -> Self
pub fn min_p(self, p: f32) -> Self
Keeps only the tokens at least p times as likely as the most likely
one, after the temperature, so the cut follows how confident the model
is from the top down. Zero keeps all of them.
Sourcepub fn presence_penalty(self, penalty: f32) -> Self
pub fn presence_penalty(self, penalty: f32) -> Self
Subtracts penalty once from the logit of every token already in the
history, however often it appears there.
Sourcepub fn frequency_penalty(self, penalty: f32) -> Self
pub fn frequency_penalty(self, penalty: f32) -> Self
Subtracts penalty from a token’s logit for every time it appears in
the history.
Sourcepub fn repetition_penalty(self, penalty: f32) -> Self
pub fn repetition_penalty(self, penalty: f32) -> Self
Divides the logit of every token already in the history by penalty,
which is how a small model is stopped from repeating one phrase for as
long as it is allowed to run. One leaves the logits alone.
Trait Implementations§
Auto Trait Implementations§
impl Forget for Sampler
impl Freeze for Sampler
impl Move for Sampler
impl RefUnwindSafe for Sampler
impl Send for Sampler
impl Sync for Sampler
impl Unpin for Sampler
impl UnsafeUnpin for Sampler
impl UnwindSafe for Sampler
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more