Struct Model 

pub struct Model {
    pub embed_tokens: Embedding,
    pub layers: Vec<DecoderLayer>,
    pub norm: RmsNormWithWeight,
    pub lm_head: Linear,
    pub rotary_emb: RotaryEmbedding,
    pub config: Config,
    /* private fields */
}

Fields

embed_tokens: Embedding

layers: Vec<DecoderLayer>

norm: RmsNormWithWeight

lm_head: Linear

rotary_emb: RotaryEmbedding

config: Config

Implementations

impl Model

pub fn load( vb: VarBuilder<'_>, cfg: &Config, dtype: DType, device: &CandleDevice, ) -> CandleResult<Self>

pub fn forward( &mut self, input_ids: &Tensor, pos: usize, cache_key: &str, ) -> CandleResult<Tensor>

pub fn clear_kv_cache_for(&mut self, cache_key: &str)

pub fn export_kv_cache( &self, cache_key: &str, ) -> Option<Vec<(Tensor, Tensor, usize, usize)>>

Export KV cache for CUDA runner migration. Returns per-layer (K_tensor, V_tensor, current_len, max_len).

pub fn release_cache(&self, _cache_key: &str)