use candle_core::{Result, Tensor};
#[derive(Clone, Debug)]
pub struct LayerKv {
pub k: Tensor,
pub v: Tensor,
}
impl LayerKv {
pub fn len(&self) -> usize {
self.k.dim(2).unwrap_or(0)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn appended(&self, k: &Tensor, v: &Tensor) -> Result<Self> {
Ok(Self {
k: Tensor::cat(&[&self.k, k], 2)?,
v: Tensor::cat(&[&self.v, v], 2)?,
})
}
pub fn trimmed(&self, max_len: usize) -> Result<Self> {
let len = self.len();
if len <= max_len {
return Ok(self.clone());
}
Ok(Self {
k: self.k.narrow(2, len - max_len, max_len)?,
v: self.v.narrow(2, len - max_len, max_len)?,
})
}
}
#[derive(Clone, Debug)]
pub struct GroupKv {
pub k: Tensor,
pub v: Tensor,
}
impl GroupKv {
pub fn len(&self) -> usize {
self.k.dim(2).unwrap_or(0)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn appended(&self, k: &Tensor, v: &Tensor) -> Result<Self> {
Ok(Self {
k: Tensor::cat(&[&self.k, k], 2)?,
v: Tensor::cat(&[&self.v, v], 2)?,
})
}
}
#[derive(Clone)]
pub struct RltState {
pub s: Tensor,
pub decoder: Vec<Option<LayerKv>>,
pub encoder: Vec<Option<LayerKv>>,
pub memory: Vec<GroupKv>,
pub position: usize,
}