use candle_core::{bail, Device, Result, Tensor, D};
pub const NORM_EPS: f32 = 1e-5;
pub fn rms_norm(x: &Tensor, eps: f32) -> Result<Tensor> {
let ms = x.powf(2.)?.mean(D::Minus1)?.unsqueeze(D::Minus1)?;
let denom = ms.broadcast_add(&Tensor::new(eps, x.device())?)?.sqrt()?;
x.broadcast_div(&denom)
}
fn rotate_half(x: &Tensor) -> Result<Tensor> {
let half = x.dim(D::Minus1)? / 2;
let x1 = x.narrow(D::Minus1, 0, half)?;
let x2 = x.narrow(D::Minus1, half, half)?.affine(-1.0, 0.0)?;
Tensor::cat(&[&x2, &x1], D::Minus1)
}
pub struct RotaryEmbedding {
cos: Tensor,
sin: Tensor,
}
impl RotaryEmbedding {
pub fn new(dim: usize, max_seq_len: usize, device: &Device) -> Result<Self> {
if dim == 0 || !dim.is_multiple_of(2) {
bail!("rotary dim must be a positive even number, got {dim}");
}
let base = 10_000f32;
let inv_freq: Vec<f32> = (0..dim / 2)
.map(|i| base.powf(-(i as f32) * 2.0 / dim as f32))
.collect();
let inv_freq = Tensor::from_vec(inv_freq, (dim / 2,), device)?;
let pos =
Tensor::arange(0u32, max_seq_len as u32, device)?.to_dtype(candle_core::DType::F32)?;
let freqs = pos.unsqueeze(1)?.broadcast_mul(&inv_freq.unsqueeze(0)?)?; Ok(Self {
cos: freqs.cos()?,
sin: freqs.sin()?,
})
}
pub fn apply(&self, x: &Tensor, offset: usize) -> Result<Tensor> {
let seq = x.dim(2)?;
let cos = self
.cos
.narrow(0, offset, seq)?
.unsqueeze(0)?
.unsqueeze(0)?;
let sin = self
.sin
.narrow(0, offset, seq)?
.unsqueeze(0)?
.unsqueeze(0)?;
let cos = Tensor::cat(&[&cos, &cos], D::Minus1)?; let sin = Tensor::cat(&[&sin, &sin], D::Minus1)?;
x.broadcast_mul(&cos)?
.broadcast_add(&rotate_half(x)?.broadcast_mul(&sin)?)
}
pub fn apply_qk(&self, q: &Tensor, k: &Tensor, offset: usize) -> Result<(Tensor, Tensor)> {
Ok((self.apply(q, offset)?, self.apply(k, offset)?))
}
}
pub fn sdpa(q: &Tensor, k: &Tensor, v: &Tensor, mask: Option<&Tensor>) -> Result<Tensor> {
let d = q.dim(D::Minus1)? as f64;
let mut scores = q.matmul(&k.transpose(D::Minus2, D::Minus1)?)?;
scores = scores.affine(1.0 / d.sqrt(), 0.0)?;
if let Some(m) = mask {
scores = scores.broadcast_add(m)?;
}
candle_nn::ops::softmax(&scores, D::Minus1)?.matmul(v)
}
pub fn causal_mask(t: usize, device: &Device) -> Result<Tensor> {
let mut rows = Vec::with_capacity(t);
for i in 0..t {
let mut row = vec![0f32; t];
for x in row.iter_mut().skip(i + 1) {
*x = f32::NEG_INFINITY;
}
rows.push(row);
}
Tensor::new(rows, device)?.unsqueeze(0)?.unsqueeze(0)
}
pub fn split_heads(x: &Tensor, n_heads: usize) -> Result<Tensor> {
let (b, s, d) = x.dims3()?;
x.reshape((b, s, n_heads, d / n_heads))?.transpose(1, 2)
}
pub fn merge_heads(x: &Tensor) -> Result<Tensor> {
let (b, h, s, hd) = x.dims4()?;
x.transpose(1, 2)?.reshape((b, s, h * hd))
}