Skip to main content

SpeculativeSampler

eredu_runtime::generation

Trait SpeculativeSampler 

Source 
pub trait SpeculativeSampler<B: SamplingBackend> {
    // Required method
    fn process_logits(
        &mut self,
        logits: &B::Logits,
        temperature: f32,
        history: &[u32],
        context: &B::Context,
    ) -> Result<B::Logits, B::Error>;

    // Provided methods
    fn uses_checkpoint_defaults(&self) -> bool { ... }
    fn supports_exact_optimistic_promotion(&self) -> bool { ... }
    fn grammar_is_complete(&mut self) -> Result<bool, B::Error> { ... }
    fn prefix_is_complete(&self, _history: &[u32]) -> Result<bool, B::Error> { ... }
    fn sample_processed(
        &self,
        logits: &B::Logits,
        temperature: f32,
        random: Option<&mut B::RandomState>,
        context: &B::Context,
    ) -> Result<B::Token, B::Error> { ... }
    fn commit_token(
        &mut self,
        _processed_logits: &B::Logits,
        _token: u32,
        _context: &B::Context,
    ) -> Result<(), B::Error> { ... }
}
Expand description

Sampling policy suitable for lossless speculative decoding.

Required Methods§

Source

fn process_logits( &mut self, logits: &B::Logits, temperature: f32, history: &[u32], context: &B::Context, ) -> Result<B::Logits, B::Error>

Applies penalties, filters, and temperature.

Provided Methods§

Source

fn uses_checkpoint_defaults(&self) -> bool

Whether loaded checkpoint defaults should wrap this policy.

Source

fn supports_exact_optimistic_promotion(&self) -> bool

Whether optimistic draft work is an exact discardable fork.

Source

fn grammar_is_complete(&mut self) -> Result<bool, B::Error>

Whether the committed generation grammar is complete.

Source

fn prefix_is_complete(&self, _history: &[u32]) -> Result<bool, B::Error>

Whether an uncommitted logical prefix completes the grammar.

Source

fn sample_processed( &self, logits: &B::Logits, temperature: f32, random: Option<&mut B::RandomState>, context: &B::Context, ) -> Result<B::Token, B::Error>

Selects from already processed logits.

Source

fn commit_token( &mut self, _processed_logits: &B::Logits, _token: u32, _context: &B::Context, ) -> Result<(), B::Error>

Commits a token selected from a processed target distribution.

Dyn Compatibility§

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors§