1use crate::error::EngineError;
7use crate::model::ModelEngine;
8use spec_stream_core::{Logits, TokenId};
9use std::collections::HashMap;
10use std::io::{Read, Seek};
11
12pub const GGUF_MAGIC: [u8; 4] = *b"GGUF";
14
15#[derive(Debug, Clone, PartialEq)]
17pub enum GgufValue {
18 Uint8(u8),
19 Int8(i8),
20 Uint16(u16),
21 Int16(i16),
22 Uint32(u32),
23 Int32(i32),
24 Float32(f32),
25 Bool(bool),
26 String(String),
27 Array(Vec<GgufValue>),
28 Uint64(u64),
29 Int64(i64),
30 Float64(f64),
31}
32
33#[derive(Debug, Clone, PartialEq, Eq)]
35pub struct GgufTensorInfo {
36 pub name: String,
37 pub dimensions: Vec<u64>,
38 pub tensor_type: u32,
39 pub offset: u64,
40}
41
42#[derive(Debug, Clone)]
44pub struct GgufMetadata {
45 pub version: u32,
46 pub tensor_count: u64,
47 pub metadata_kv_count: u64,
48 pub metadata: HashMap<String, GgufValue>,
49 pub tensors: Vec<GgufTensorInfo>,
50 pub data_offset: u64,
51}
52
53impl GgufMetadata {
54 pub fn parse<R: Read + Seek>(reader: &mut R) -> Result<Self, EngineError> {
56 let mut magic = [0u8; 4];
57 reader
58 .read_exact(&mut magic)
59 .map_err(|e| EngineError::Inference(format!("Failed to read GGUF magic: {}", e)))?;
60
61 if magic != GGUF_MAGIC {
62 return Err(EngineError::Inference(format!(
63 "Invalid GGUF magic: expected {:?}, got {:?}",
64 GGUF_MAGIC, magic
65 )));
66 }
67
68 let mut buf4 = [0u8; 4];
69 let mut buf8 = [0u8; 8];
70
71 reader
72 .read_exact(&mut buf4)
73 .map_err(|e| EngineError::Inference(e.to_string()))?;
74 let version = u32::from_le_bytes(buf4);
75 if !(2..=3).contains(&version) {
76 return Err(EngineError::Inference(format!(
77 "Unsupported GGUF version: {}",
78 version
79 )));
80 }
81
82 reader
83 .read_exact(&mut buf8)
84 .map_err(|e| EngineError::Inference(e.to_string()))?;
85 let tensor_count = u64::from_le_bytes(buf8);
86
87 reader
88 .read_exact(&mut buf8)
89 .map_err(|e| EngineError::Inference(e.to_string()))?;
90 let metadata_kv_count = u64::from_le_bytes(buf8);
91
92 let mut metadata = HashMap::with_capacity(metadata_kv_count as usize);
93
94 for _ in 0..metadata_kv_count {
95 let key = read_gguf_string(reader)?;
96 let val = read_gguf_value(reader)?;
97 metadata.insert(key, val);
98 }
99
100 let mut tensors = Vec::with_capacity(tensor_count as usize);
101 for _ in 0..tensor_count {
102 let name = read_gguf_string(reader)?;
103 reader
104 .read_exact(&mut buf4)
105 .map_err(|e| EngineError::Inference(e.to_string()))?;
106 let n_dims = u32::from_le_bytes(buf4);
107
108 let mut dims = Vec::with_capacity(n_dims as usize);
109 for _ in 0..n_dims {
110 reader
111 .read_exact(&mut buf8)
112 .map_err(|e| EngineError::Inference(e.to_string()))?;
113 dims.push(u64::from_le_bytes(buf8));
114 }
115
116 reader
117 .read_exact(&mut buf4)
118 .map_err(|e| EngineError::Inference(e.to_string()))?;
119 let tensor_type = u32::from_le_bytes(buf4);
120
121 reader
122 .read_exact(&mut buf8)
123 .map_err(|e| EngineError::Inference(e.to_string()))?;
124 let offset = u64::from_le_bytes(buf8);
125
126 tensors.push(GgufTensorInfo {
127 name,
128 dimensions: dims,
129 tensor_type,
130 offset,
131 });
132 }
133
134 let pos = reader
135 .stream_position()
136 .map_err(|e| EngineError::Inference(e.to_string()))?;
137 let alignment = match metadata.get("general.alignment") {
138 Some(GgufValue::Uint32(a)) => *a as u64,
139 Some(GgufValue::Uint64(a)) => *a,
140 _ => 32,
141 };
142 let remainder = pos % alignment;
143 let data_offset = if remainder == 0 {
144 pos
145 } else {
146 pos + (alignment - remainder)
147 };
148
149 Ok(Self {
150 version,
151 tensor_count,
152 metadata_kv_count,
153 metadata,
154 tensors,
155 data_offset,
156 })
157 }
158
159 pub fn get_string(&self, key: &str) -> Option<&str> {
161 match self.metadata.get(key) {
162 Some(GgufValue::String(s)) => Some(s.as_str()),
163 _ => None,
164 }
165 }
166
167 pub fn get_u64(&self, key: &str) -> Option<u64> {
169 match self.metadata.get(key) {
170 Some(GgufValue::Uint64(v)) => Some(*v),
171 Some(GgufValue::Uint32(v)) => Some(*v as u64),
172 Some(GgufValue::Int64(v)) if *v >= 0 => Some(*v as u64),
173 _ => None,
174 }
175 }
176}
177
178fn read_gguf_string<R: Read>(reader: &mut R) -> Result<String, EngineError> {
179 let mut buf8 = [0u8; 8];
180 reader
181 .read_exact(&mut buf8)
182 .map_err(|e| EngineError::Inference(e.to_string()))?;
183 let len = u64::from_le_bytes(buf8) as usize;
184
185 let mut str_bytes = vec![0u8; len];
186 reader
187 .read_exact(&mut str_bytes)
188 .map_err(|e| EngineError::Inference(e.to_string()))?;
189
190 String::from_utf8(str_bytes)
191 .map_err(|e| EngineError::Inference(format!("GGUF invalid UTF-8 string: {}", e)))
192}
193
194fn read_gguf_value<R: Read>(reader: &mut R) -> Result<GgufValue, EngineError> {
195 let mut buf4 = [0u8; 4];
196 reader
197 .read_exact(&mut buf4)
198 .map_err(|e| EngineError::Inference(e.to_string()))?;
199 let val_type = u32::from_le_bytes(buf4);
200
201 read_gguf_value_typed(reader, val_type)
202}
203
204fn read_gguf_value_typed<R: Read>(reader: &mut R, val_type: u32) -> Result<GgufValue, EngineError> {
205 let mut buf1 = [0u8; 1];
206 let mut buf2 = [0u8; 2];
207 let mut buf4 = [0u8; 4];
208 let mut buf8 = [0u8; 8];
209
210 match val_type {
211 0 => {
212 reader
213 .read_exact(&mut buf1)
214 .map_err(|e| EngineError::Inference(e.to_string()))?;
215 Ok(GgufValue::Uint8(buf1[0]))
216 }
217 1 => {
218 reader
219 .read_exact(&mut buf1)
220 .map_err(|e| EngineError::Inference(e.to_string()))?;
221 Ok(GgufValue::Int8(buf1[0] as i8))
222 }
223 2 => {
224 reader
225 .read_exact(&mut buf2)
226 .map_err(|e| EngineError::Inference(e.to_string()))?;
227 Ok(GgufValue::Uint16(u16::from_le_bytes(buf2)))
228 }
229 3 => {
230 reader
231 .read_exact(&mut buf2)
232 .map_err(|e| EngineError::Inference(e.to_string()))?;
233 Ok(GgufValue::Int16(i16::from_le_bytes(buf2)))
234 }
235 4 => {
236 reader
237 .read_exact(&mut buf4)
238 .map_err(|e| EngineError::Inference(e.to_string()))?;
239 Ok(GgufValue::Uint32(u32::from_le_bytes(buf4)))
240 }
241 5 => {
242 reader
243 .read_exact(&mut buf4)
244 .map_err(|e| EngineError::Inference(e.to_string()))?;
245 Ok(GgufValue::Int32(i32::from_le_bytes(buf4)))
246 }
247 6 => {
248 reader
249 .read_exact(&mut buf4)
250 .map_err(|e| EngineError::Inference(e.to_string()))?;
251 Ok(GgufValue::Float32(f32::from_le_bytes(buf4)))
252 }
253 7 => {
254 reader
255 .read_exact(&mut buf1)
256 .map_err(|e| EngineError::Inference(e.to_string()))?;
257 Ok(GgufValue::Bool(buf1[0] != 0))
258 }
259 8 => {
260 let s = read_gguf_string(reader)?;
261 Ok(GgufValue::String(s))
262 }
263 9 => {
264 reader
265 .read_exact(&mut buf4)
266 .map_err(|e| EngineError::Inference(e.to_string()))?;
267 let elem_type = u32::from_le_bytes(buf4);
268 reader
269 .read_exact(&mut buf8)
270 .map_err(|e| EngineError::Inference(e.to_string()))?;
271 let count = u64::from_le_bytes(buf8) as usize;
272
273 let mut arr = Vec::with_capacity(count);
274 for _ in 0..count {
275 arr.push(read_gguf_value_typed(reader, elem_type)?);
276 }
277 Ok(GgufValue::Array(arr))
278 }
279 10 => {
280 reader
281 .read_exact(&mut buf8)
282 .map_err(|e| EngineError::Inference(e.to_string()))?;
283 Ok(GgufValue::Uint64(u64::from_le_bytes(buf8)))
284 }
285 11 => {
286 reader
287 .read_exact(&mut buf8)
288 .map_err(|e| EngineError::Inference(e.to_string()))?;
289 Ok(GgufValue::Int64(i64::from_le_bytes(buf8)))
290 }
291 12 => {
292 reader
293 .read_exact(&mut buf8)
294 .map_err(|e| EngineError::Inference(e.to_string()))?;
295 Ok(GgufValue::Float64(f64::from_le_bytes(buf8)))
296 }
297 other => Err(EngineError::Inference(format!(
298 "Unsupported GGUF value type ID: {}",
299 other
300 ))),
301 }
302}
303
304#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
306pub struct SafeTensorEntry {
307 pub dtype: String,
308 pub shape: Vec<usize>,
309 pub data_offsets: [usize; 2],
310}
311
312#[derive(Debug, Clone)]
314pub struct SafeTensorsMetadata {
315 pub tensors: HashMap<String, SafeTensorEntry>,
316 pub data_offset: usize,
317}
318
319impl SafeTensorsMetadata {
320 pub fn parse(bytes: &[u8]) -> Result<Self, EngineError> {
322 if bytes.len() < 8 {
323 return Err(EngineError::Inference(
324 "SafeTensors buffer too short for 8-byte header length".into(),
325 ));
326 }
327
328 let header_len = u64::from_le_bytes(bytes[0..8].try_into().unwrap()) as usize;
329 let data_offset = 8 + header_len;
330
331 if bytes.len() < data_offset {
332 return Err(EngineError::Inference(
333 "SafeTensors header exceeds buffer length".into(),
334 ));
335 }
336
337 let header_json = std::str::from_utf8(&bytes[8..data_offset]).map_err(|e| {
338 EngineError::Inference(format!("SafeTensors header not valid UTF-8: {}", e))
339 })?;
340
341 let tensors: HashMap<String, SafeTensorEntry> = serde_json::from_str(header_json)
342 .map_err(|e| EngineError::Inference(format!("SafeTensors JSON invalid: {}", e)))?;
343
344 Ok(Self {
345 tensors,
346 data_offset,
347 })
348 }
349}
350
351#[derive(Debug, Clone)]
356pub struct LinearProjectionModel {
357 pub name: String,
358 pub vocab_size: usize,
359 pub hidden_dim: usize,
360 pub weights: Vec<f32>,
362 pub embeddings: Vec<f32>,
364}
365
366impl LinearProjectionModel {
367 pub fn new(name: impl Into<String>, vocab_size: usize, hidden_dim: usize) -> Self {
369 let size = vocab_size * hidden_dim;
370 let mut weights = Vec::with_capacity(size);
371 let mut embeddings = Vec::with_capacity(size);
372
373 let inv = 1.0 / (hidden_dim as f32).sqrt();
375 for i in 0..size {
376 let val = ((i % 17) as f32 - 8.0) * inv * 0.1;
377 weights.push(val);
378 embeddings.push(val);
379 }
380
381 Self {
382 name: name.into(),
383 vocab_size,
384 hidden_dim,
385 weights,
386 embeddings,
387 }
388 }
389
390 pub fn project_hidden(&self, hidden: &[f32]) -> Logits {
392 assert_eq!(
393 hidden.len(),
394 self.hidden_dim,
395 "Hidden dimension mismatch in projection"
396 );
397 let mut logits = Vec::with_capacity(self.vocab_size);
398
399 for row in 0..self.vocab_size {
400 let offset = row * self.hidden_dim;
401 let mut dot = 0.0f32;
402 for (col, &h) in hidden.iter().enumerate().take(self.hidden_dim) {
403 dot += h * self.weights[offset + col];
404 }
405 logits.push(dot);
406 }
407
408 Logits::new(logits)
409 }
410
411 pub fn project_token(&self, token: TokenId) -> Logits {
413 let token_idx = (token as usize) % self.vocab_size;
414 let emb_offset = token_idx * self.hidden_dim;
415 let hidden = &self.embeddings[emb_offset..emb_offset + self.hidden_dim];
416 self.project_hidden(hidden)
417 }
418}
419
420impl ModelEngine for LinearProjectionModel {
421 async fn forward_batch(&mut self, tokens: &[TokenId]) -> Result<Vec<Logits>, EngineError> {
422 let mut out = Vec::with_capacity(tokens.len());
423 for &tok in tokens {
424 out.push(self.project_token(tok));
425 }
426 Ok(out)
427 }
428
429 async fn forward_token(&mut self, token: TokenId) -> Result<Logits, EngineError> {
430 Ok(self.project_token(token))
431 }
432}
433
434#[cfg(test)]
435mod tests {
436 use super::*;
437 use std::io::Cursor;
438
439 #[test]
440 fn test_linear_projection_model_forward() {
441 let model = LinearProjectionModel::new("test-linear", 16, 8);
442 assert_eq!(model.vocab_size, 16);
443 assert_eq!(model.hidden_dim, 8);
444
445 let logits = model.project_token(3);
446 assert_eq!(logits.vocab_size(), 16);
447 }
448
449 #[test]
450 fn test_safetensors_metadata_parser() {
451 let json_header =
452 r#"{"lm_head.weight":{"dtype":"F32","shape":[16,8],"data_offsets":[0,512]}}"#;
453 let mut buffer = Vec::new();
454 let header_len = json_header.len() as u64;
455 buffer.extend_from_slice(&header_len.to_le_bytes());
456 buffer.extend_from_slice(json_header.as_bytes());
457 buffer.resize(buffer.len() + 512, 0u8);
458
459 let parsed = SafeTensorsMetadata::parse(&buffer).expect("Must parse SafeTensors");
460 assert_eq!(parsed.tensors.len(), 1);
461 let entry = &parsed.tensors["lm_head.weight"];
462 assert_eq!(entry.dtype, "F32");
463 assert_eq!(entry.shape, vec![16, 8]);
464 assert_eq!(entry.data_offsets, [0, 512]);
465 }
466
467 #[test]
468 fn test_gguf_metadata_parser_synthetic() {
469 let mut buf = Vec::new();
470 buf.extend_from_slice(b"GGUF");
472 buf.extend_from_slice(&3u32.to_le_bytes());
474 buf.extend_from_slice(&1u64.to_le_bytes());
476 buf.extend_from_slice(&2u64.to_le_bytes());
478
479 let k1 = "general.architecture";
481 buf.extend_from_slice(&(k1.len() as u64).to_le_bytes());
482 buf.extend_from_slice(k1.as_bytes());
483 buf.extend_from_slice(&8u32.to_le_bytes()); let v1 = "llama";
485 buf.extend_from_slice(&(v1.len() as u64).to_le_bytes());
486 buf.extend_from_slice(v1.as_bytes());
487
488 let k2 = "llama.context_length";
490 buf.extend_from_slice(&(k2.len() as u64).to_le_bytes());
491 buf.extend_from_slice(k2.as_bytes());
492 buf.extend_from_slice(&4u32.to_le_bytes()); buf.extend_from_slice(&4096u32.to_le_bytes());
494
495 let t1 = "token_embd.weight";
497 buf.extend_from_slice(&(t1.len() as u64).to_le_bytes());
498 buf.extend_from_slice(t1.as_bytes());
499 buf.extend_from_slice(&2u32.to_le_bytes()); buf.extend_from_slice(&4096u64.to_le_bytes());
501 buf.extend_from_slice(&32000u64.to_le_bytes());
502 buf.extend_from_slice(&0u32.to_le_bytes()); buf.extend_from_slice(&0u64.to_le_bytes()); let mut cursor = Cursor::new(buf);
506 let meta = GgufMetadata::parse(&mut cursor).expect("GGUF parse failed");
507
508 assert_eq!(meta.version, 3);
509 assert_eq!(meta.tensor_count, 1);
510 assert_eq!(meta.get_string("general.architecture"), Some("llama"));
511 assert_eq!(meta.get_u64("llama.context_length"), Some(4096));
512 assert_eq!(meta.tensors.len(), 1);
513 assert_eq!(meta.tensors[0].name, "token_embd.weight");
514 assert_eq!(meta.tensors[0].dimensions, vec![4096, 32000]);
515 }
516}