oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
use serde::{Deserialize, Serialize};

fn default_beta_fast() -> f32 {
    32.0
}

fn default_beta_slow() -> f32 {
    1.0
}

fn is_default_beta_fast(value: &f32) -> bool {
    *value == default_beta_fast()
}

fn is_default_beta_slow(value: &f32) -> bool {
    *value == default_beta_slow()
}

/// Frequency scaling used to extend RoPE beyond a model's training context.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct RopeScaling {
    #[serde(rename = "rope_type", alias = "type")]
    pub kind: RopeScalingType,
    pub factor: f32,
    #[serde(default)]
    pub original_max_position_embeddings: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub attention_factor: Option<f32>,
    #[serde(
        default = "default_beta_fast",
        skip_serializing_if = "is_default_beta_fast"
    )]
    pub beta_fast: f32,
    #[serde(
        default = "default_beta_slow",
        skip_serializing_if = "is_default_beta_slow"
    )]
    pub beta_slow: f32,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RopeScalingType {
    Linear,
    Yarn,
}

impl RopeScaling {
    pub(crate) fn infer_original_context(&mut self, max_sequence_length: usize) {
        if self.original_max_position_embeddings == 0
            && self.factor.is_finite()
            && self.factor > 0.0
        {
            self.original_max_position_embeddings =
                (max_sequence_length as f64 / self.factor as f64).round() as usize;
        }
    }

    pub fn validate(&self) -> Result<(), String> {
        if !self.factor.is_finite() || self.factor <= 1.0 {
            return Err("RoPE scaling factor must be finite and greater than 1".into());
        }
        if self.original_max_position_embeddings == 0 {
            return Err("RoPE original context length must be positive".into());
        }
        if self
            .attention_factor
            .is_some_and(|factor| !factor.is_finite() || factor <= 0.0)
        {
            return Err("RoPE attention factor must be positive and finite".into());
        }
        if self.kind == RopeScalingType::Yarn {
            if !self.beta_fast.is_finite()
                || !self.beta_slow.is_finite()
                || self.beta_fast <= self.beta_slow
                || self.beta_slow <= 0.0
            {
                return Err("YaRN beta_fast must be greater than positive beta_slow".into());
            }
        }
        Ok(())
    }

    pub(crate) fn kind_id(&self) -> u32 {
        match self.kind {
            RopeScalingType::Linear => 1,
            RopeScalingType::Yarn => 2,
        }
    }

    pub(crate) fn attention_factor(&self) -> f32 {
        match self.kind {
            RopeScalingType::Linear => 1.0,
            RopeScalingType::Yarn => self
                .attention_factor
                .unwrap_or_else(|| 0.1 * self.factor.ln() + 1.0),
        }
    }

    pub(crate) fn frequency(&self, feature: usize, width: usize, theta: f32) -> f32 {
        let base_frequency = theta.powf(-(feature as f32) / width as f32);
        match self.kind {
            RopeScalingType::Linear => base_frequency / self.factor,
            RopeScalingType::Yarn => {
                let low = correction_dimension(
                    self.beta_fast,
                    width,
                    theta,
                    self.original_max_position_embeddings,
                )
                .floor()
                .max(0.0);
                let high = correction_dimension(
                    self.beta_slow,
                    width,
                    theta,
                    self.original_max_position_embeddings,
                )
                .ceil()
                .min((width - 1) as f32);
                let pair = (feature / 2) as f32;
                let ramp = if low == high {
                    ((pair - low) / 0.001).clamp(0.0, 1.0)
                } else {
                    ((pair - low) / (high - low)).clamp(0.0, 1.0)
                };
                base_frequency * ((1.0 - ramp) + ramp / self.factor)
            }
        }
    }
}

fn correction_dimension(
    rotations: f32,
    width: usize,
    theta: f32,
    original_max_position_embeddings: usize,
) -> f32 {
    let context = original_max_position_embeddings as f32;
    width as f32 * (context / (rotations * 2.0 * std::f32::consts::PI)).ln() / (2.0 * theta.ln())
}

#[cfg(test)]
mod tests {
    use super::{RopeScaling, RopeScalingType};

    #[test]
    fn accepts_hugging_face_rope_type_and_legacy_type() {
        for key in ["rope_type", "type"] {
            let json = format!(
                r#"{{"{key}":"linear","factor":2.0,"original_max_position_embeddings":2048}}"#
            );
            let scaling: RopeScaling = serde_json::from_str(&json).unwrap();
            assert_eq!(scaling.kind, RopeScalingType::Linear);
        }
    }
}