use burn::{
Tensor,
prelude::Int,
tensor::Device,
};
use super::CachedHistory;
use crate::{
blocks::transformers::{
decoders::stack::{
DecoderStackCache,
DecoderStackMeta,
},
models::language_decoder::{
LmDecoderOps,
LmPlainStackOps,
},
},
kits::support::decoding::{
CachedStep,
StackedLm,
},
ops::transformers::attention::KVCacheStorage,
};
pub trait CachedLm: CachedStep {
fn decode_state(
&self,
rows: usize,
positions: usize,
storage: KVCacheStorage,
device: &Device,
) -> Self::State;
fn prefill_ids(
&self,
ids: Tensor<2, Int>,
state: &mut Self::State,
) -> Tensor<2>;
fn rewind<'a>(
&self,
history: &mut CachedHistory,
state: &mut Self::State,
next: &'a [i64],
device: &Device,
) -> &'a [i64];
fn rewinds_anywhere(&self) -> bool {
true
}
fn snapshot(
&self,
_state: &mut Self::State,
) {
}
}
impl<M: StackedLm> CachedLm for M {
fn decode_state(
&self,
rows: usize,
positions: usize,
storage: KVCacheStorage,
device: &Device,
) -> DecoderStackCache {
self.stack().decode_cache(rows, positions, storage, device)
}
fn prefill_ids(
&self,
ids: Tensor<2, Int>,
state: &mut DecoderStackCache,
) -> Tensor<2> {
self.prefill(LmDecoderOps::embed(self, ids), state)
}
fn rewind<'a>(
&self,
history: &mut CachedHistory,
state: &mut DecoderStackCache,
next: &'a [i64],
_device: &Device,
) -> &'a [i64] {
history.rewind(state, next)
}
fn rewinds_anywhere(&self) -> bool {
LmPlainStackOps::stack(self)
.layer_attends()
.iter()
.all(|&attends| attends)
}
fn snapshot(
&self,
state: &mut DecoderStackCache,
) {
state.snapshot();
}
}