use serde::{Deserialize, Serialize};
fn default_beta_fast() -> f32 {
32.0
}
fn default_beta_slow() -> f32 {
1.0
}
fn is_default_beta_fast(value: &f32) -> bool {
*value == default_beta_fast()
}
fn is_default_beta_slow(value: &f32) -> bool {
*value == default_beta_slow()
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct RopeScaling {
#[serde(rename = "rope_type", alias = "type")]
pub kind: RopeScalingType,
pub factor: f32,
#[serde(default)]
pub original_max_position_embeddings: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub attention_factor: Option<f32>,
#[serde(
default = "default_beta_fast",
skip_serializing_if = "is_default_beta_fast"
)]
pub beta_fast: f32,
#[serde(
default = "default_beta_slow",
skip_serializing_if = "is_default_beta_slow"
)]
pub beta_slow: f32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RopeScalingType {
Linear,
Yarn,
}
impl RopeScaling {
pub(crate) fn infer_original_context(&mut self, max_sequence_length: usize) {
if self.original_max_position_embeddings == 0
&& self.factor.is_finite()
&& self.factor > 0.0
{
self.original_max_position_embeddings =
(max_sequence_length as f64 / self.factor as f64).round() as usize;
}
}
pub fn validate(&self) -> Result<(), String> {
if !self.factor.is_finite() || self.factor <= 1.0 {
return Err("RoPE scaling factor must be finite and greater than 1".into());
}
if self.original_max_position_embeddings == 0 {
return Err("RoPE original context length must be positive".into());
}
if self
.attention_factor
.is_some_and(|factor| !factor.is_finite() || factor <= 0.0)
{
return Err("RoPE attention factor must be positive and finite".into());
}
if self.kind == RopeScalingType::Yarn {
if !self.beta_fast.is_finite()
|| !self.beta_slow.is_finite()
|| self.beta_fast <= self.beta_slow
|| self.beta_slow <= 0.0
{
return Err("YaRN beta_fast must be greater than positive beta_slow".into());
}
}
Ok(())
}
pub(crate) fn kind_id(&self) -> u32 {
match self.kind {
RopeScalingType::Linear => 1,
RopeScalingType::Yarn => 2,
}
}
pub(crate) fn attention_factor(&self) -> f32 {
match self.kind {
RopeScalingType::Linear => 1.0,
RopeScalingType::Yarn => self
.attention_factor
.unwrap_or_else(|| 0.1 * self.factor.ln() + 1.0),
}
}
pub(crate) fn frequency(&self, feature: usize, width: usize, theta: f32) -> f32 {
let base_frequency = theta.powf(-(feature as f32) / width as f32);
match self.kind {
RopeScalingType::Linear => base_frequency / self.factor,
RopeScalingType::Yarn => {
let low = correction_dimension(
self.beta_fast,
width,
theta,
self.original_max_position_embeddings,
)
.floor()
.max(0.0);
let high = correction_dimension(
self.beta_slow,
width,
theta,
self.original_max_position_embeddings,
)
.ceil()
.min((width - 1) as f32);
let pair = (feature / 2) as f32;
let ramp = if low == high {
((pair - low) / 0.001).clamp(0.0, 1.0)
} else {
((pair - low) / (high - low)).clamp(0.0, 1.0)
};
base_frequency * ((1.0 - ramp) + ramp / self.factor)
}
}
}
}
fn correction_dimension(
rotations: f32,
width: usize,
theta: f32,
original_max_position_embeddings: usize,
) -> f32 {
let context = original_max_position_embeddings as f32;
width as f32 * (context / (rotations * 2.0 * std::f32::consts::PI)).ln() / (2.0 * theta.ln())
}
#[cfg(test)]
mod tests {
use super::{RopeScaling, RopeScalingType};
#[test]
fn accepts_hugging_face_rope_type_and_legacy_type() {
for key in ["rope_type", "type"] {
let json = format!(
r#"{{"{key}":"linear","factor":2.0,"original_max_position_embeddings":2048}}"#
);
let scaling: RopeScaling = serde_json::from_str(&json).unwrap();
assert_eq!(scaling.kind, RopeScalingType::Linear);
}
}
}