Struct ScatteredCacheBuilder

pub struct ScatteredCacheBuilder { /* private fields */ }

Implementations

impl ScatteredCacheBuilder

pub fn new( batch_size: usize, context: usize, dtype: DType, device: &Device, ) -> Result<Self>

pub fn make_cache( &self, num_heads: usize, head_dim: usize, ) -> Result<ScatteredKvCache>

pub fn positions(&self) -> &[usize]

pub fn reset(&mut self)

pub fn batch_size(&self) -> usize

pub fn reset_batch_index(&mut self, batch_index: usize)

pub fn indices_and_mask( &mut self, seq_len: usize, batch_mask: &[bool], ) -> Result<IndicesAndMask>

pub fn device(&self) -> &Device

Trait Implementations

impl Clone for ScatteredCacheBuilder

fn clone(&self) -> ScatteredCacheBuilder

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ScatteredCacheBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.