ferrum-quantization 0.7.7

Weight-format abstraction (Dense / GPTQ / AWQ / GGUF) for Ferrum models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

//! LoRA reference utilities.
//!
//! G4 keeps production adapter serving startup-scoped. This module provides
//! the small f32 reference path used by loader and routing tests:
//!
//! y = base(x) + (alpha / r) * B(A(x))

use ferrum_types::{FerrumError, Result};

#[derive(Debug, Clone)]
pub struct LoraLinearRef {
    base_weight: Vec<f32>,
    a_weight: Vec<f32>,
    b_weight: Vec<f32>,
    in_features: usize,
    out_features: usize,
    rank: usize,
    scaling: f32,
}

impl LoraLinearRef {
    pub fn new(
        base_weight: Vec<f32>,
        a_weight: Vec<f32>,
        b_weight: Vec<f32>,
        in_features: usize,
        out_features: usize,
        rank: usize,
        lora_alpha: f32,
    ) -> Result<Self> {
        if rank == 0 {
            return Err(FerrumError::config("LoRA rank must be > 0"));
        }
        if base_weight.len() != out_features * in_features {
            return Err(FerrumError::config(format!(
                "base weight shape mismatch: got {} elements, expected {}x{}",
                base_weight.len(),
                out_features,
                in_features
            )));
        }
        if a_weight.len() != rank * in_features {
            return Err(FerrumError::config(format!(
                "LoRA A shape mismatch: got {} elements, expected {}x{}",
                a_weight.len(),
                rank,
                in_features
            )));
        }
        if b_weight.len() != out_features * rank {
            return Err(FerrumError::config(format!(
                "LoRA B shape mismatch: got {} elements, expected {}x{}",
                b_weight.len(),
                out_features,
                rank
            )));
        }
        Ok(Self {
            base_weight,
            a_weight,
            b_weight,
            in_features,
            out_features,
            rank,
            scaling: lora_alpha / rank as f32,
        })
    }

    pub fn forward(&self, input: &[f32], batch: usize) -> Result<Vec<f32>> {
        if input.len() != batch * self.in_features {
            return Err(FerrumError::config(format!(
                "LoRA input shape mismatch: got {} elements, expected {}x{}",
                input.len(),
                batch,
                self.in_features
            )));
        }
        let mut out = vec![0.0f32; batch * self.out_features];
        let mut low_rank = vec![0.0f32; batch * self.rank];

        for m in 0..batch {
            for o in 0..self.out_features {
                let mut acc = 0.0f32;
                for i in 0..self.in_features {
                    acc += input[m * self.in_features + i]
                        * self.base_weight[o * self.in_features + i];
                }
                out[m * self.out_features + o] = acc;
            }
            for r in 0..self.rank {
                let mut acc = 0.0f32;
                for i in 0..self.in_features {
                    acc +=
                        input[m * self.in_features + i] * self.a_weight[r * self.in_features + i];
                }
                low_rank[m * self.rank + r] = acc;
            }
            for o in 0..self.out_features {
                let mut acc = 0.0f32;
                for r in 0..self.rank {
                    acc += low_rank[m * self.rank + r] * self.b_weight[o * self.rank + r];
                }
                out[m * self.out_features + o] += self.scaling * acc;
            }
        }

        Ok(out)
    }
}