Struct LlamaKVCacheInterface 

pub struct LlamaKVCacheInterface { /* private fields */ }

Interface for interacting with llama.cpp’s KV cache

Implementations

impl LlamaKVCacheInterface

pub fn new() -> Self

pub fn set_runtime(&mut self, runtime: Box<dyn ModelRuntime + Send>)

pub async fn get_current_cache_state(&self) -> Result<LlamaKVCacheState>

Get current KV cache state from llama.cpp

pub async fn extract_current_kv_entries(&self) -> Result<Vec<KVEntry>>

Extract current KV cache entries from llama.cpp

pub async fn inject_kv_entries(&self, entries: &[KVEntry]) -> Result<()>

Inject KV cache entries back into llama.cpp

pub async fn clear_cache_entries( &self, layer_indices: &[i32], head_indices: &[Option<i32>], ) -> Result<()>

Clear specific entries from llama.cpp’s KV cache

pub async fn get_cache_memory_usage(&self) -> Result<usize>

Calculate memory usage of current KV cache

pub async fn estimate_cache_capacity(&self) -> Result<f32>

Estimate when the KV cache will reach capacity

Trait Implementations

impl Default for LlamaKVCacheInterface

fn default() -> Self

Returns the “default value” for a type.