Skip to main content

spec_stream_engine/
weights.rs

1//! # Model Weights & Metadata Parsers
2//!
3//! Provides zero-copy / streaming parsers for GGUF (v2/v3) binary headers and
4//! SafeTensors metadata, alongside lightweight weight loaders for neural model runtimes.
5
6use crate::error::EngineError;
7use crate::model::ModelEngine;
8use spec_stream_core::{Logits, TokenId};
9use std::collections::HashMap;
10use std::io::{Read, Seek};
11
12/// Magic identifier for GGUF files (`b"GGUF"`).
13pub const GGUF_MAGIC: [u8; 4] = *b"GGUF";
14
15/// Value type in GGUF metadata key-value pairs.
16#[derive(Debug, Clone, PartialEq)]
17pub enum GgufValue {
18    Uint8(u8),
19    Int8(i8),
20    Uint16(u16),
21    Int16(i16),
22    Uint32(u32),
23    Int32(i32),
24    Float32(f32),
25    Bool(bool),
26    String(String),
27    Array(Vec<GgufValue>),
28    Uint64(u64),
29    Int64(i64),
30    Float64(f64),
31}
32
33/// Metadata describing a single tensor stored inside a GGUF file.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub struct GgufTensorInfo {
36    pub name: String,
37    pub dimensions: Vec<u64>,
38    pub tensor_type: u32,
39    pub offset: u64,
40}
41
42/// Parsed header and metadata of a GGUF model file.
43#[derive(Debug, Clone)]
44pub struct GgufMetadata {
45    pub version: u32,
46    pub tensor_count: u64,
47    pub metadata_kv_count: u64,
48    pub metadata: HashMap<String, GgufValue>,
49    pub tensors: Vec<GgufTensorInfo>,
50    pub data_offset: u64,
51}
52
53impl GgufMetadata {
54    /// Parses a GGUF header from any reader with seek capabilities.
55    pub fn parse<R: Read + Seek>(reader: &mut R) -> Result<Self, EngineError> {
56        let mut magic = [0u8; 4];
57        reader
58            .read_exact(&mut magic)
59            .map_err(|e| EngineError::Inference(format!("Failed to read GGUF magic: {}", e)))?;
60
61        if magic != GGUF_MAGIC {
62            return Err(EngineError::Inference(format!(
63                "Invalid GGUF magic: expected {:?}, got {:?}",
64                GGUF_MAGIC, magic
65            )));
66        }
67
68        let mut buf4 = [0u8; 4];
69        let mut buf8 = [0u8; 8];
70
71        reader
72            .read_exact(&mut buf4)
73            .map_err(|e| EngineError::Inference(e.to_string()))?;
74        let version = u32::from_le_bytes(buf4);
75        if !(2..=3).contains(&version) {
76            return Err(EngineError::Inference(format!(
77                "Unsupported GGUF version: {}",
78                version
79            )));
80        }
81
82        reader
83            .read_exact(&mut buf8)
84            .map_err(|e| EngineError::Inference(e.to_string()))?;
85        let tensor_count = u64::from_le_bytes(buf8);
86
87        reader
88            .read_exact(&mut buf8)
89            .map_err(|e| EngineError::Inference(e.to_string()))?;
90        let metadata_kv_count = u64::from_le_bytes(buf8);
91
92        let mut metadata = HashMap::with_capacity(metadata_kv_count as usize);
93
94        for _ in 0..metadata_kv_count {
95            let key = read_gguf_string(reader)?;
96            let val = read_gguf_value(reader)?;
97            metadata.insert(key, val);
98        }
99
100        let mut tensors = Vec::with_capacity(tensor_count as usize);
101        for _ in 0..tensor_count {
102            let name = read_gguf_string(reader)?;
103            reader
104                .read_exact(&mut buf4)
105                .map_err(|e| EngineError::Inference(e.to_string()))?;
106            let n_dims = u32::from_le_bytes(buf4);
107
108            let mut dims = Vec::with_capacity(n_dims as usize);
109            for _ in 0..n_dims {
110                reader
111                    .read_exact(&mut buf8)
112                    .map_err(|e| EngineError::Inference(e.to_string()))?;
113                dims.push(u64::from_le_bytes(buf8));
114            }
115
116            reader
117                .read_exact(&mut buf4)
118                .map_err(|e| EngineError::Inference(e.to_string()))?;
119            let tensor_type = u32::from_le_bytes(buf4);
120
121            reader
122                .read_exact(&mut buf8)
123                .map_err(|e| EngineError::Inference(e.to_string()))?;
124            let offset = u64::from_le_bytes(buf8);
125
126            tensors.push(GgufTensorInfo {
127                name,
128                dimensions: dims,
129                tensor_type,
130                offset,
131            });
132        }
133
134        let pos = reader
135            .stream_position()
136            .map_err(|e| EngineError::Inference(e.to_string()))?;
137        let alignment = match metadata.get("general.alignment") {
138            Some(GgufValue::Uint32(a)) => *a as u64,
139            Some(GgufValue::Uint64(a)) => *a,
140            _ => 32,
141        };
142        let remainder = pos % alignment;
143        let data_offset = if remainder == 0 {
144            pos
145        } else {
146            pos + (alignment - remainder)
147        };
148
149        Ok(Self {
150            version,
151            tensor_count,
152            metadata_kv_count,
153            metadata,
154            tensors,
155            data_offset,
156        })
157    }
158
159    /// Retrieves an optional string parameter from GGUF metadata.
160    pub fn get_string(&self, key: &str) -> Option<&str> {
161        match self.metadata.get(key) {
162            Some(GgufValue::String(s)) => Some(s.as_str()),
163            _ => None,
164        }
165    }
166
167    /// Retrieves an optional u64 parameter from GGUF metadata.
168    pub fn get_u64(&self, key: &str) -> Option<u64> {
169        match self.metadata.get(key) {
170            Some(GgufValue::Uint64(v)) => Some(*v),
171            Some(GgufValue::Uint32(v)) => Some(*v as u64),
172            Some(GgufValue::Int64(v)) if *v >= 0 => Some(*v as u64),
173            _ => None,
174        }
175    }
176}
177
178fn read_gguf_string<R: Read>(reader: &mut R) -> Result<String, EngineError> {
179    let mut buf8 = [0u8; 8];
180    reader
181        .read_exact(&mut buf8)
182        .map_err(|e| EngineError::Inference(e.to_string()))?;
183    let len = u64::from_le_bytes(buf8) as usize;
184
185    let mut str_bytes = vec![0u8; len];
186    reader
187        .read_exact(&mut str_bytes)
188        .map_err(|e| EngineError::Inference(e.to_string()))?;
189
190    String::from_utf8(str_bytes)
191        .map_err(|e| EngineError::Inference(format!("GGUF invalid UTF-8 string: {}", e)))
192}
193
194fn read_gguf_value<R: Read>(reader: &mut R) -> Result<GgufValue, EngineError> {
195    let mut buf4 = [0u8; 4];
196    reader
197        .read_exact(&mut buf4)
198        .map_err(|e| EngineError::Inference(e.to_string()))?;
199    let val_type = u32::from_le_bytes(buf4);
200
201    read_gguf_value_typed(reader, val_type)
202}
203
204fn read_gguf_value_typed<R: Read>(reader: &mut R, val_type: u32) -> Result<GgufValue, EngineError> {
205    let mut buf1 = [0u8; 1];
206    let mut buf2 = [0u8; 2];
207    let mut buf4 = [0u8; 4];
208    let mut buf8 = [0u8; 8];
209
210    match val_type {
211        0 => {
212            reader
213                .read_exact(&mut buf1)
214                .map_err(|e| EngineError::Inference(e.to_string()))?;
215            Ok(GgufValue::Uint8(buf1[0]))
216        }
217        1 => {
218            reader
219                .read_exact(&mut buf1)
220                .map_err(|e| EngineError::Inference(e.to_string()))?;
221            Ok(GgufValue::Int8(buf1[0] as i8))
222        }
223        2 => {
224            reader
225                .read_exact(&mut buf2)
226                .map_err(|e| EngineError::Inference(e.to_string()))?;
227            Ok(GgufValue::Uint16(u16::from_le_bytes(buf2)))
228        }
229        3 => {
230            reader
231                .read_exact(&mut buf2)
232                .map_err(|e| EngineError::Inference(e.to_string()))?;
233            Ok(GgufValue::Int16(i16::from_le_bytes(buf2)))
234        }
235        4 => {
236            reader
237                .read_exact(&mut buf4)
238                .map_err(|e| EngineError::Inference(e.to_string()))?;
239            Ok(GgufValue::Uint32(u32::from_le_bytes(buf4)))
240        }
241        5 => {
242            reader
243                .read_exact(&mut buf4)
244                .map_err(|e| EngineError::Inference(e.to_string()))?;
245            Ok(GgufValue::Int32(i32::from_le_bytes(buf4)))
246        }
247        6 => {
248            reader
249                .read_exact(&mut buf4)
250                .map_err(|e| EngineError::Inference(e.to_string()))?;
251            Ok(GgufValue::Float32(f32::from_le_bytes(buf4)))
252        }
253        7 => {
254            reader
255                .read_exact(&mut buf1)
256                .map_err(|e| EngineError::Inference(e.to_string()))?;
257            Ok(GgufValue::Bool(buf1[0] != 0))
258        }
259        8 => {
260            let s = read_gguf_string(reader)?;
261            Ok(GgufValue::String(s))
262        }
263        9 => {
264            reader
265                .read_exact(&mut buf4)
266                .map_err(|e| EngineError::Inference(e.to_string()))?;
267            let elem_type = u32::from_le_bytes(buf4);
268            reader
269                .read_exact(&mut buf8)
270                .map_err(|e| EngineError::Inference(e.to_string()))?;
271            let count = u64::from_le_bytes(buf8) as usize;
272
273            let mut arr = Vec::with_capacity(count);
274            for _ in 0..count {
275                arr.push(read_gguf_value_typed(reader, elem_type)?);
276            }
277            Ok(GgufValue::Array(arr))
278        }
279        10 => {
280            reader
281                .read_exact(&mut buf8)
282                .map_err(|e| EngineError::Inference(e.to_string()))?;
283            Ok(GgufValue::Uint64(u64::from_le_bytes(buf8)))
284        }
285        11 => {
286            reader
287                .read_exact(&mut buf8)
288                .map_err(|e| EngineError::Inference(e.to_string()))?;
289            Ok(GgufValue::Int64(i64::from_le_bytes(buf8)))
290        }
291        12 => {
292            reader
293                .read_exact(&mut buf8)
294                .map_err(|e| EngineError::Inference(e.to_string()))?;
295            Ok(GgufValue::Float64(f64::from_le_bytes(buf8)))
296        }
297        other => Err(EngineError::Inference(format!(
298            "Unsupported GGUF value type ID: {}",
299            other
300        ))),
301    }
302}
303
304/// Metadata description for SafeTensors files.
305#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
306pub struct SafeTensorEntry {
307    pub dtype: String,
308    pub shape: Vec<usize>,
309    pub data_offsets: [usize; 2],
310}
311
312/// Parsed SafeTensors header.
313#[derive(Debug, Clone)]
314pub struct SafeTensorsMetadata {
315    pub tensors: HashMap<String, SafeTensorEntry>,
316    pub data_offset: usize,
317}
318
319impl SafeTensorsMetadata {
320    /// Parses SafeTensors header from bytes.
321    pub fn parse(bytes: &[u8]) -> Result<Self, EngineError> {
322        if bytes.len() < 8 {
323            return Err(EngineError::Inference(
324                "SafeTensors buffer too short for 8-byte header length".into(),
325            ));
326        }
327
328        let header_len = u64::from_le_bytes(bytes[0..8].try_into().unwrap()) as usize;
329        let data_offset = 8 + header_len;
330
331        if bytes.len() < data_offset {
332            return Err(EngineError::Inference(
333                "SafeTensors header exceeds buffer length".into(),
334            ));
335        }
336
337        let header_json = std::str::from_utf8(&bytes[8..data_offset]).map_err(|e| {
338            EngineError::Inference(format!("SafeTensors header not valid UTF-8: {}", e))
339        })?;
340
341        let tensors: HashMap<String, SafeTensorEntry> = serde_json::from_str(header_json)
342            .map_err(|e| EngineError::Inference(format!("SafeTensors JSON invalid: {}", e)))?;
343
344        Ok(Self {
345            tensors,
346            data_offset,
347        })
348    }
349}
350
351/// A lightweight, real linear projection model engine for fast embeddings and head inference.
352///
353/// Multiplies an input token embedding against an output projection matrix `W` of shape `[vocab_size, hidden_dim]`.
354/// Supports direct loading of weights for speculative drafting.
355#[derive(Debug, Clone)]
356pub struct LinearProjectionModel {
357    pub name: String,
358    pub vocab_size: usize,
359    pub hidden_dim: usize,
360    /// Weights matrix of shape `[vocab_size, hidden_dim]` in row-major order.
361    pub weights: Vec<f32>,
362    /// Embedding lookup table `[vocab_size, hidden_dim]`.
363    pub embeddings: Vec<f32>,
364}
365
366impl LinearProjectionModel {
367    /// Creates a new projection model with initial weights.
368    pub fn new(name: impl Into<String>, vocab_size: usize, hidden_dim: usize) -> Self {
369        let size = vocab_size * hidden_dim;
370        let mut weights = Vec::with_capacity(size);
371        let mut embeddings = Vec::with_capacity(size);
372
373        // Simple normalized initialization
374        let inv = 1.0 / (hidden_dim as f32).sqrt();
375        for i in 0..size {
376            let val = ((i % 17) as f32 - 8.0) * inv * 0.1;
377            weights.push(val);
378            embeddings.push(val);
379        }
380
381        Self {
382            name: name.into(),
383            vocab_size,
384            hidden_dim,
385            weights,
386            embeddings,
387        }
388    }
389
390    /// Computes logits from hidden states via linear projection: `logits = hidden * W^T`.
391    pub fn project_hidden(&self, hidden: &[f32]) -> Logits {
392        assert_eq!(
393            hidden.len(),
394            self.hidden_dim,
395            "Hidden dimension mismatch in projection"
396        );
397        let mut logits = Vec::with_capacity(self.vocab_size);
398
399        for row in 0..self.vocab_size {
400            let offset = row * self.hidden_dim;
401            let mut dot = 0.0f32;
402            for (col, &h) in hidden.iter().enumerate().take(self.hidden_dim) {
403                dot += h * self.weights[offset + col];
404            }
405            logits.push(dot);
406        }
407
408        Logits::new(logits)
409    }
410
411    /// Computes logits directly for an input token using its embedded representation.
412    pub fn project_token(&self, token: TokenId) -> Logits {
413        let token_idx = (token as usize) % self.vocab_size;
414        let emb_offset = token_idx * self.hidden_dim;
415        let hidden = &self.embeddings[emb_offset..emb_offset + self.hidden_dim];
416        self.project_hidden(hidden)
417    }
418}
419
420impl ModelEngine for LinearProjectionModel {
421    async fn forward_batch(&mut self, tokens: &[TokenId]) -> Result<Vec<Logits>, EngineError> {
422        let mut out = Vec::with_capacity(tokens.len());
423        for &tok in tokens {
424            out.push(self.project_token(tok));
425        }
426        Ok(out)
427    }
428
429    async fn forward_token(&mut self, token: TokenId) -> Result<Logits, EngineError> {
430        Ok(self.project_token(token))
431    }
432}
433
434#[cfg(test)]
435mod tests {
436    use super::*;
437    use std::io::Cursor;
438
439    #[test]
440    fn test_linear_projection_model_forward() {
441        let model = LinearProjectionModel::new("test-linear", 16, 8);
442        assert_eq!(model.vocab_size, 16);
443        assert_eq!(model.hidden_dim, 8);
444
445        let logits = model.project_token(3);
446        assert_eq!(logits.vocab_size(), 16);
447    }
448
449    #[test]
450    fn test_safetensors_metadata_parser() {
451        let json_header =
452            r#"{"lm_head.weight":{"dtype":"F32","shape":[16,8],"data_offsets":[0,512]}}"#;
453        let mut buffer = Vec::new();
454        let header_len = json_header.len() as u64;
455        buffer.extend_from_slice(&header_len.to_le_bytes());
456        buffer.extend_from_slice(json_header.as_bytes());
457        buffer.resize(buffer.len() + 512, 0u8);
458
459        let parsed = SafeTensorsMetadata::parse(&buffer).expect("Must parse SafeTensors");
460        assert_eq!(parsed.tensors.len(), 1);
461        let entry = &parsed.tensors["lm_head.weight"];
462        assert_eq!(entry.dtype, "F32");
463        assert_eq!(entry.shape, vec![16, 8]);
464        assert_eq!(entry.data_offsets, [0, 512]);
465    }
466
467    #[test]
468    fn test_gguf_metadata_parser_synthetic() {
469        let mut buf = Vec::new();
470        // Magic
471        buf.extend_from_slice(b"GGUF");
472        // Version 3
473        buf.extend_from_slice(&3u32.to_le_bytes());
474        // Tensor count = 1
475        buf.extend_from_slice(&1u64.to_le_bytes());
476        // Metadata KV count = 2
477        buf.extend_from_slice(&2u64.to_le_bytes());
478
479        // KV 1: "general.architecture" -> String "llama"
480        let k1 = "general.architecture";
481        buf.extend_from_slice(&(k1.len() as u64).to_le_bytes());
482        buf.extend_from_slice(k1.as_bytes());
483        buf.extend_from_slice(&8u32.to_le_bytes()); // Type 8 = String
484        let v1 = "llama";
485        buf.extend_from_slice(&(v1.len() as u64).to_le_bytes());
486        buf.extend_from_slice(v1.as_bytes());
487
488        // KV 2: "llama.context_length" -> Uint32 4096
489        let k2 = "llama.context_length";
490        buf.extend_from_slice(&(k2.len() as u64).to_le_bytes());
491        buf.extend_from_slice(k2.as_bytes());
492        buf.extend_from_slice(&4u32.to_le_bytes()); // Type 4 = Uint32
493        buf.extend_from_slice(&4096u32.to_le_bytes());
494
495        // Tensor 1: "token_embd.weight", n_dims=2, dims=[4096, 32000], type=0, offset=0
496        let t1 = "token_embd.weight";
497        buf.extend_from_slice(&(t1.len() as u64).to_le_bytes());
498        buf.extend_from_slice(t1.as_bytes());
499        buf.extend_from_slice(&2u32.to_le_bytes()); // 2 dims
500        buf.extend_from_slice(&4096u64.to_le_bytes());
501        buf.extend_from_slice(&32000u64.to_le_bytes());
502        buf.extend_from_slice(&0u32.to_le_bytes()); // F32
503        buf.extend_from_slice(&0u64.to_le_bytes()); // offset
504
505        let mut cursor = Cursor::new(buf);
506        let meta = GgufMetadata::parse(&mut cursor).expect("GGUF parse failed");
507
508        assert_eq!(meta.version, 3);
509        assert_eq!(meta.tensor_count, 1);
510        assert_eq!(meta.get_string("general.architecture"), Some("llama"));
511        assert_eq!(meta.get_u64("llama.context_length"), Some(4096));
512        assert_eq!(meta.tensors.len(), 1);
513        assert_eq!(meta.tensors[0].name, "token_embd.weight");
514        assert_eq!(meta.tensors[0].dimensions, vec![4096, 32000]);
515    }
516}