pub struct PromptCache { /* private fields */ }Expand description
Prompt cache for prefix sharing
Implementations§
Source§impl PromptCache
impl PromptCache
Sourcepub fn new(config: PromptCacheConfig) -> Self
pub fn new(config: PromptCacheConfig) -> Self
Create a new prompt cache
Sourcepub fn cache_prefix(
&mut self,
tokens: &[u32],
k_cache: Vec<Tensor>,
v_cache: Vec<Tensor>,
) -> PrefixId
pub fn cache_prefix( &mut self, tokens: &[u32], k_cache: Vec<Tensor>, v_cache: Vec<Tensor>, ) -> PrefixId
Cache a prefix
Sourcepub fn get_prefix(&mut self, id: &PrefixId) -> Option<&CachedPrefix>
pub fn get_prefix(&mut self, id: &PrefixId) -> Option<&CachedPrefix>
Get a cached prefix
Sourcepub fn find_matching_prefix(
&mut self,
tokens: &[u32],
) -> Option<(PrefixId, usize)>
pub fn find_matching_prefix( &mut self, tokens: &[u32], ) -> Option<(PrefixId, usize)>
Find the longest matching prefix
Sourcepub fn remove_prefix(&mut self, id: &PrefixId)
pub fn remove_prefix(&mut self, id: &PrefixId)
Remove a prefix from cache
Sourcepub fn stats(&self) -> PromptCacheStats
pub fn stats(&self) -> PromptCacheStats
Get cache statistics
Sourcepub fn release_prefix(&mut self, id: &PrefixId)
pub fn release_prefix(&mut self, id: &PrefixId)
Decrease reference count for a prefix
Auto Trait Implementations§
impl Freeze for PromptCache
impl RefUnwindSafe for PromptCache
impl Send for PromptCache
impl Sync for PromptCache
impl Unpin for PromptCache
impl UnsafeUnpin for PromptCache
impl UnwindSafe for PromptCache
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more