Struct PagedSpeculativeCacheAccess 

pub struct PagedSpeculativeCacheAccess<'a> { /* private fields */ }

Implementations

impl<'a> PagedSpeculativeCacheAccess<'a>

pub fn new(metadata: &'a PagedAttentionMeta, cache_engine: &CacheEngine) -> Self

Trait Implementations

impl<'a> SpeculativeCacheAccess for PagedSpeculativeCacheAccess<'a>

type Guard = PagedSpeculativeCacheGuard<'a>

fn begin( &self, seq_id: usize, base_len: usize, verify_len: usize, ) -> Result<Option<Self::Guard>>

Returns Ok(None) when the cache cannot reserve speculative slots and the caller should fall back to normal decoding for this step.

fn guard_for_reserved( &self, seq_id: usize, base_len: usize, verify_len: usize, ) -> PagedSpeculativeCacheGuard<'a>

fn make_verify_input_metadata( &self, verify_tokens: &[u32], seq_id: usize, base_len: usize, device: &Device, mapper: &dyn DeviceMapper, ) -> Result<InputMetadata>

fn proposer_cache( &self, _sequences: &[&Sequence], ) -> Result<SpeculativeKvCache<'_>>

fn finish_verification( &self, guard: &mut Self::Guard, _seq: &mut Sequence, keep_len: usize, accepted_all: bool, ) -> Result<()>

fn finish_verification_batch( &self, guards: &mut [Option<Self::Guard>], seqs: &mut [&mut Sequence], outcomes: &[Option<SpeculativeCacheOutcome>], ) -> Result<()>

fn can_stage_proposal( &self, _sequences: &[&Sequence], _base_lens: &[usize], _proposal_len: usize, ) -> bool