1use std::io;
16use std::path::{Path, PathBuf};
17use std::sync::{Arc, Mutex};
18
19use tpt_gpu_primitives::memory::{BufferFlags, DType, Shape};
20use tpt_gpu_primitives::{
21 AttentionKernel, EmbeddingKernel, GemmKernel, GpuBuffer, QuantGemmKernel, RmsNormKernel,
22 SoftmaxKernel, VendorBackend,
23};
24
25use crate::arch::{template_for_arch, ArchTemplate, ForwardOp};
26use crate::error::{ErrorCode, TptrError, TptrResult};
27use crate::kv_cache::KvCache;
28use crate::rope::RopeConfig;
29
30#[derive(Debug, Clone)]
36pub struct ModelInfo {
37 pub path: PathBuf,
38 pub arch: String,
40 pub context_len: u32,
42 pub vocab_size: u32,
44 pub hidden_dim: u32,
46 pub num_heads: u32,
48 pub num_kv_heads: u32,
50 pub ffn_dim: u32,
52 pub num_layers: u32,
54 pub per_layer_bits: Vec<u8>,
57 pub pruning_mask: Option<Vec<Vec<u32>>>,
60}
61
62#[derive(Debug, Clone, Copy, PartialEq)]
64pub enum ModelFormat {
65 Gguf,
66 Tptf,
67 Unknown,
68}
69
70pub fn detect_format(path: &Path) -> ModelFormat {
72 use std::fs::File;
73 use std::io::Read;
74 let Ok(mut f) = File::open(path) else {
75 return ModelFormat::Unknown;
76 };
77 let mut magic = [0u8; 4];
78 if f.read_exact(&mut magic).is_err() {
79 return ModelFormat::Unknown;
80 }
81 match &magic {
82 b"GGUF" => ModelFormat::Gguf,
83 b"TPTF" => ModelFormat::Tptf,
84 _ => ModelFormat::Unknown,
85 }
86}
87
88#[repr(u32)]
94#[derive(Debug, Clone, Copy, PartialEq)]
95enum GgufType {
96 Uint8 = 0,
97 Int8 = 1,
98 Uint16 = 2,
99 Int16 = 3,
100 Uint32 = 4,
101 Int32 = 5,
102 Float32 = 6,
103 Bool = 7,
104 String = 8,
105 Array = 9,
106 Uint64 = 10,
107 Int64 = 11,
108 Float64 = 12,
109}
110
111impl GgufType {
112 fn from_u32(v: u32) -> Option<Self> {
113 match v {
114 0 => Some(Self::Uint8),
115 1 => Some(Self::Int8),
116 2 => Some(Self::Uint16),
117 3 => Some(Self::Int16),
118 4 => Some(Self::Uint32),
119 5 => Some(Self::Int32),
120 6 => Some(Self::Float32),
121 7 => Some(Self::Bool),
122 8 => Some(Self::String),
123 9 => Some(Self::Array),
124 10 => Some(Self::Uint64),
125 11 => Some(Self::Int64),
126 12 => Some(Self::Float64),
127 _ => None,
128 }
129 }
130
131 fn scalar_bytes(self) -> Option<usize> {
133 match self {
134 Self::Bool | Self::Uint8 | Self::Int8 => Some(1),
135 Self::Uint16 | Self::Int16 => Some(2),
136 Self::Uint32 | Self::Int32 | Self::Float32 => Some(4),
137 Self::Uint64 | Self::Int64 | Self::Float64 => Some(8),
138 Self::String | Self::Array => None,
139 }
140 }
141}
142
143struct BufReader<'a> {
145 data: &'a [u8],
146 pos: usize,
147}
148
149impl<'a> BufReader<'a> {
150 fn new(data: &'a [u8]) -> Self {
151 Self { data, pos: 0 }
152 }
153
154 fn remaining(&self) -> usize {
155 self.data.len().saturating_sub(self.pos)
156 }
157
158 fn read_bytes(&mut self, n: usize) -> io::Result<&'a [u8]> {
159 if self.pos + n > self.data.len() {
160 return Err(io::Error::new(
161 io::ErrorKind::UnexpectedEof,
162 "GGUF truncated",
163 ));
164 }
165 let slice = &self.data[self.pos..self.pos + n];
166 self.pos += n;
167 Ok(slice)
168 }
169
170 fn read_u8(&mut self) -> io::Result<u8> {
171 Ok(self.read_bytes(1)?[0])
172 }
173 #[allow(dead_code)]
174 fn read_u16(&mut self) -> io::Result<u16> {
175 Ok(u16::from_le_bytes(self.read_bytes(2)?.try_into().unwrap()))
176 }
177 fn read_u32(&mut self) -> io::Result<u32> {
178 Ok(u32::from_le_bytes(self.read_bytes(4)?.try_into().unwrap()))
179 }
180 #[allow(dead_code)]
181 fn read_i32(&mut self) -> io::Result<i32> {
182 Ok(i32::from_le_bytes(self.read_bytes(4)?.try_into().unwrap()))
183 }
184 fn read_u64(&mut self) -> io::Result<u64> {
185 Ok(u64::from_le_bytes(self.read_bytes(8)?.try_into().unwrap()))
186 }
187 #[allow(dead_code)]
188 fn read_f32(&mut self) -> io::Result<f32> {
189 Ok(f32::from_le_bytes(self.read_bytes(4)?.try_into().unwrap()))
190 }
191 #[allow(dead_code)]
192 fn read_bool(&mut self) -> io::Result<bool> {
193 Ok(self.read_u8()? != 0)
194 }
195
196 fn read_gguf_string(&mut self) -> io::Result<String> {
197 let len = self.read_u64()? as usize;
198 let bytes = self.read_bytes(len)?;
199 String::from_utf8(bytes.to_vec()).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
200 }
201
202 fn skip_value(&mut self, vtype: GgufType) -> io::Result<()> {
204 match vtype {
205 GgufType::String => {
206 self.read_gguf_string()?;
207 }
208 GgufType::Array => {
209 let elem_type = GgufType::from_u32(self.read_u32()?).ok_or_else(|| {
210 io::Error::new(io::ErrorKind::InvalidData, "unknown array elem type")
211 })?;
212 let count = self.read_u64()? as usize;
213 for _ in 0..count {
214 self.skip_value(elem_type)?;
215 }
216 }
217 scalar => {
218 let n = scalar.scalar_bytes().ok_or_else(|| {
219 io::Error::new(io::ErrorKind::InvalidData, "scalar_bytes unset")
220 })?;
221 self.read_bytes(n)?;
222 }
223 }
224 Ok(())
225 }
226}
227
228pub fn parse_gguf_header(path: &Path) -> TptrResult<ModelInfo> {
233 use std::fs;
234
235 if !path.exists() {
236 return Err(TptrError::new(
237 ErrorCode::InvalidKernel,
238 format!("model file not found: {}", path.display()),
239 ));
240 }
241
242 let data = fs::read(path).map_err(|e| {
243 TptrError::new(
244 ErrorCode::InternalError,
245 format!("cannot read {}: {}", path.display(), e),
246 )
247 })?;
248
249 let mut r = BufReader::new(&data);
250
251 let magic = r.read_bytes(4).map_err(|e| {
253 TptrError::new(
254 ErrorCode::InvalidKernel,
255 format!("cannot read GGUF magic: {}", e),
256 )
257 })?;
258 if magic != b"GGUF" {
259 return Err(TptrError::new(
260 ErrorCode::InvalidKernel,
261 format!("not a GGUF file (magic = {:?})", magic),
262 ));
263 }
264
265 let version = r.read_u32().map_err(|e| {
267 TptrError::new(
268 ErrorCode::InvalidKernel,
269 format!("cannot read GGUF version: {}", e),
270 )
271 })?;
272 if version == 0 || version > 3 {
273 return Err(TptrError::new(
274 ErrorCode::InvalidKernel,
275 format!("unsupported GGUF version {} (supported: 1, 2, 3)", version),
276 ));
277 }
278
279 let (_tensor_count, kv_count) = if version >= 2 {
281 (r.read_u64().unwrap_or(0), r.read_u64().unwrap_or(0))
282 } else {
283 (
284 r.read_u32().unwrap_or(0) as u64,
285 r.read_u32().unwrap_or(0) as u64,
286 )
287 };
288
289 let mut arch = String::new();
291 let mut context_len: u32 = 2048;
292 let mut hidden_dim: u32 = 0;
293 let mut num_heads: u32 = 0;
294 let mut num_kv_heads: u32 = 0;
295 let mut ffn_dim: u32 = 0;
296 let mut num_layers: u32 = 0;
297 let mut vocab_size: u32 = 0;
298
299 for _ in 0..kv_count {
300 let key = match r.read_gguf_string() {
301 Ok(k) => k,
302 Err(_) => break,
303 };
304 let vtype_raw = match r.read_u32() {
305 Ok(v) => v,
306 Err(_) => break,
307 };
308 let vtype = match GgufType::from_u32(vtype_raw) {
309 Some(t) => t,
310 None => break,
311 };
312
313 match key.as_str() {
314 "general.architecture" if vtype == GgufType::String => {
315 arch = r.read_gguf_string().unwrap_or_default();
316 }
317 "llm.context_length" if vtype == GgufType::Uint32 => {
318 context_len = r.read_u32().unwrap_or(2048);
319 }
320 "llm.embedding_length" if vtype == GgufType::Uint32 => {
321 hidden_dim = r.read_u32().unwrap_or(0);
322 }
323 "llm.attention.head_count" if vtype == GgufType::Uint32 => {
324 num_heads = r.read_u32().unwrap_or(0);
325 }
326 "llm.attention.head_count_kv" if vtype == GgufType::Uint32 => {
327 num_kv_heads = r.read_u32().unwrap_or(0);
328 }
329 "llm.feed_forward_length" if vtype == GgufType::Uint32 => {
330 ffn_dim = r.read_u32().unwrap_or(0);
331 }
332 "llm.block_count" if vtype == GgufType::Uint32 => {
333 num_layers = r.read_u32().unwrap_or(0);
334 }
335 "tokenizer.ggml.tokens" if vtype == GgufType::Array => {
336 let _elem_type = r.read_u32().unwrap_or(0);
338 let count = r.read_u64().unwrap_or(0);
339 vocab_size = count as u32;
340 let elem_t = GgufType::String; for _ in 0..count {
343 let _ = r.skip_value(elem_t);
344 }
345 }
346 _ => {
347 if r.skip_value(vtype).is_err() {
348 break;
349 }
350 }
351 }
352 }
353
354 if hidden_dim == 0 {
356 hidden_dim = 4096;
357 }
358 if num_heads == 0 {
359 num_heads = 32;
360 }
361 if num_layers == 0 {
362 num_layers = 32;
363 }
364 if vocab_size == 0 {
365 vocab_size = 32000;
366 }
367 if ffn_dim == 0 {
368 ffn_dim = hidden_dim * 8 / 3;
369 } Ok(ModelInfo {
372 path: path.to_owned(),
373 arch,
374 context_len,
375 vocab_size,
376 hidden_dim,
377 num_heads,
378 num_kv_heads,
379 ffn_dim,
380 num_layers,
381 per_layer_bits: Vec::new(),
382 pruning_mask: None,
383 })
384}
385
386pub fn parse_tptf_header(path: &Path) -> TptrResult<ModelInfo> {
406 use std::fs;
407
408 if !path.exists() {
409 return Err(TptrError::new(
410 ErrorCode::InvalidKernel,
411 format!("model file not found: {}", path.display()),
412 ));
413 }
414
415 let data = fs::read(path).map_err(|e| {
416 TptrError::new(
417 ErrorCode::InternalError,
418 format!("cannot read {}: {}", path.display(), e),
419 )
420 })?;
421
422 if data.len() < 512 {
423 return Err(TptrError::new(
424 ErrorCode::InvalidKernel,
425 format!("TPTF file too short ({} bytes, need ≥ 512)", data.len()),
426 ));
427 }
428
429 let mut r = BufReader::new(&data);
430
431 let magic = r.read_bytes(4).map_err(|e| {
432 TptrError::new(
433 ErrorCode::InvalidKernel,
434 format!("cannot read TPTF magic: {}", e),
435 )
436 })?;
437 if magic != b"TPTF" {
438 return Err(TptrError::new(
439 ErrorCode::InvalidKernel,
440 format!("not a TPTF file (magic = {:?})", magic),
441 ));
442 }
443
444 let version = r.read_u32().map_err(|e| {
445 TptrError::new(
446 ErrorCode::InvalidKernel,
447 format!("cannot read TPTF version: {}", e),
448 )
449 })?;
450 if version != 1 {
451 return Err(TptrError::new(
452 ErrorCode::InvalidKernel,
453 format!("unsupported TPTF version {} (supported: 1)", version),
454 ));
455 }
456
457 let flags = r.read_u32().unwrap_or(0);
458 let has_pruning_mask = (flags & 1) != 0;
459
460 let arch_len = r.read_u8().unwrap_or(0) as usize;
462 let arch = if arch_len > 0 && arch_len <= 63 {
463 let bytes = r.read_bytes(arch_len).unwrap_or_default();
464 String::from_utf8_lossy(bytes).to_string()
465 } else {
466 r.read_bytes(63.min(r.remaining())).ok(); String::new()
468 };
469 let consumed = 4 + 4 + 4 + 1 + arch_len;
471 if consumed < 76 {
472 let _ = r.read_bytes(76 - consumed);
473 }
474
475 let context_len = r.read_u32().unwrap_or(2048);
476 let vocab_size = r.read_u32().unwrap_or(32000);
477 let hidden_dim = r.read_u32().unwrap_or(4096);
478 let num_heads = r.read_u32().unwrap_or(32);
479 let num_kv_heads = r.read_u32().unwrap_or(8);
480 let ffn_dim = r.read_u32().unwrap_or(0);
481 let num_layers = r.read_u32().unwrap_or(32);
482
483 let bits_bytes = r.read_bytes(128).unwrap_or_default();
485 let per_layer_bits: Vec<u8> = bits_bytes[..num_layers.min(128) as usize]
486 .iter()
487 .copied()
488 .filter(|&b| b > 0)
489 .collect();
490
491 let pruning_mask = if has_pruning_mask {
492 Some(vec![Vec::new(); num_layers as usize])
493 } else {
494 None
495 };
496
497 let ffn_dim = if ffn_dim == 0 {
498 hidden_dim * 8 / 3
499 } else {
500 ffn_dim
501 };
502
503 Ok(ModelInfo {
504 path: path.to_owned(),
505 arch,
506 context_len,
507 vocab_size,
508 hidden_dim,
509 num_heads,
510 num_kv_heads,
511 ffn_dim,
512 num_layers,
513 per_layer_bits,
514 pruning_mask,
515 })
516}
517
518struct LayerWeights {
524 q_proj: GpuBuffer<f32>, k_proj: GpuBuffer<f32>, v_proj: GpuBuffer<f32>, o_proj: GpuBuffer<f32>, gate_proj: GpuBuffer<f32>, up_proj: GpuBuffer<f32>, down_proj: GpuBuffer<f32>, norm1: GpuBuffer<f32>, norm2: GpuBuffer<f32>, }
534
535struct ModelWeights {
538 embed_table: GpuBuffer<f32>, layers: Vec<LayerWeights>,
540 final_norm: GpuBuffer<f32>, lm_head: GpuBuffer<f32>, lm_head_tied: bool,
543}
544
545fn alloc_f32(rows: usize, cols: usize) -> TptrResult<GpuBuffer<f32>> {
546 GpuBuffer::new(Shape::dim2(rows, cols), DType::F32, BufferFlags::STORAGE)
547 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))
548}
549
550fn alloc_f32_1d(n: usize) -> TptrResult<GpuBuffer<f32>> {
551 GpuBuffer::new(Shape::new(&[n]), DType::F32, BufferFlags::STORAGE)
552 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))
553}
554
555impl ModelWeights {
556 fn load_tptf(path: &Path, info: &ModelInfo, tied: bool) -> TptrResult<Self> {
566 use std::fs;
567
568 let data = fs::read(path).map_err(|e| {
569 TptrError::new(
570 ErrorCode::InternalError,
571 format!("load_tptf: cannot read {}: {}", path.display(), e),
572 )
573 })?;
574
575 const TPTF_TENSOR_OFFSET_FIELD: usize = 460;
578 const TPTF_HEADER_SIZE: usize = 512;
579 let tensor_start: usize = if data.len() >= TPTF_TENSOR_OFFSET_FIELD + 8 {
580 let raw = u64::from_le_bytes(
581 data[TPTF_TENSOR_OFFSET_FIELD..TPTF_TENSOR_OFFSET_FIELD + 8]
582 .try_into()
583 .unwrap(),
584 ) as usize;
585 if raw >= TPTF_HEADER_SIZE {
586 raw
587 } else {
588 TPTF_HEADER_SIZE
589 }
590 } else {
591 TPTF_HEADER_SIZE
592 };
593
594 let tokenizer_start: usize = if data.len() >= TPTF_TENSOR_OFFSET_FIELD + 16 {
596 u64::from_le_bytes(
597 data[TPTF_TENSOR_OFFSET_FIELD + 8..TPTF_TENSOR_OFFSET_FIELD + 16]
598 .try_into()
599 .unwrap(),
600 ) as usize
601 } else {
602 0
603 };
604
605 let tensor_end = if tokenizer_start > tensor_start {
606 tokenizer_start
607 } else {
608 data.len()
609 };
610
611 let mut weights = Self::allocate(info, tied)?;
612 let num_layers = info.num_layers as usize;
613
614 let mut pos = tensor_start;
630 while pos + 14 < tensor_end.min(data.len()) {
631 let block_start = pos;
632
633 if pos + 4 > data.len() {
635 break;
636 }
637 let layer_idx = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
638 pos += 4;
639
640 if pos >= data.len() {
642 break;
643 }
644 let name_len = data[pos] as usize;
645 pos += 1;
646 if name_len > 31 || pos + name_len > data.len() {
647 break;
648 }
649 let name = String::from_utf8_lossy(&data[pos..pos + name_len]).to_string();
650 pos += name_len;
651
652 if pos + 13 > data.len() {
654 break;
655 }
656 let bits = data[pos];
657 pos += 1;
658 let group_size = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
659 pos += 4;
660 let rows = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
661 pos += 4;
662 let cols = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
663 pos += 4;
664
665 if pos + 4 > data.len() {
667 break;
668 }
669 let weights_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
670 pos += 4;
671 if pos + weights_len > data.len() {
672 break;
673 }
674 let packed = data[pos..pos + weights_len].to_vec();
675 pos += weights_len;
676
677 if pos + 4 > data.len() {
679 break;
680 }
681 let scales_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
682 pos += 4;
683 if pos + scales_len * 4 > data.len() {
684 break;
685 }
686 let mut scales = vec![1.0f32; scales_len];
687 for s in scales.iter_mut() {
688 *s = f32::from_le_bytes(data[pos..pos + 4].try_into().unwrap());
689 pos += 4;
690 }
691
692 if pos + 4 > data.len() {
694 break;
695 }
696 let zp_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
697 pos += 4;
698 if pos + zp_len > data.len() {
699 break;
700 }
701 let zero_points: Vec<i8> = data[pos..pos + zp_len].iter().map(|&b| b as i8).collect();
702 pos += zp_len;
703
704 let _ = block_start; pos = (pos + 127) & !127_usize;
707
708 let num_elements = rows * cols;
710 let group_sz = if group_size > 0 {
711 group_size
712 } else {
713 num_elements.max(1)
714 };
715 let f32_data =
716 tptf_dequantize(&packed, &scales, &zero_points, bits, group_sz, num_elements);
717
718 if layer_idx < num_layers {
720 let layer = &mut weights.layers[layer_idx];
721 let target: Option<&mut GpuBuffer<f32>> = match name.as_str() {
722 "gate_proj" => Some(&mut layer.gate_proj),
723 "up_proj" => Some(&mut layer.up_proj),
724 "down_proj" => Some(&mut layer.down_proj),
725 "q_proj" => Some(&mut layer.q_proj),
726 "k_proj" => Some(&mut layer.k_proj),
727 "v_proj" => Some(&mut layer.v_proj),
728 "o_proj" => Some(&mut layer.o_proj),
729 "norm1" => Some(&mut layer.norm1),
730 "norm2" => Some(&mut layer.norm2),
731 _ => None,
732 };
733 if let Some(buf) = target {
734 let n = f32_data.len().min(buf.num_elements());
735 if n > 0 {
736 let _ = buf.copy_from_host(&f32_data[..n]);
737 }
738 }
739 } else if name == "embed_table" || name == "embedding" {
740 let n = f32_data.len().min(weights.embed_table.num_elements());
741 if n > 0 {
742 let _ = weights.embed_table.copy_from_host(&f32_data[..n]);
743 }
744 } else if name == "final_norm" {
745 let n = f32_data.len().min(weights.final_norm.num_elements());
746 if n > 0 {
747 let _ = weights.final_norm.copy_from_host(&f32_data[..n]);
748 }
749 } else if name == "lm_head" && !weights.lm_head_tied {
750 let n = f32_data.len().min(weights.lm_head.num_elements());
751 if n > 0 {
752 let _ = weights.lm_head.copy_from_host(&f32_data[..n]);
753 }
754 }
755 }
756
757 Ok(weights)
758 }
759
760 fn allocate(info: &ModelInfo, tied: bool) -> TptrResult<Self> {
761 let h = info.hidden_dim as usize;
762 let v = info.vocab_size as usize;
763 let nh = info.num_heads.max(1) as usize;
764 let nk = info.num_kv_heads.max(1) as usize;
765 let hd = h / nh; let f = info.ffn_dim as usize;
767
768 let embed_table = alloc_f32(v, h)?;
769 let final_norm = alloc_f32_1d(h)?;
770 let lm_head = if tied {
771 alloc_f32(1, 1)?
772 } else {
773 alloc_f32(h, v)?
774 }; let mut layers = Vec::with_capacity(info.num_layers as usize);
777 for _ in 0..info.num_layers {
778 layers.push(LayerWeights {
779 q_proj: alloc_f32(h, nh * hd)?,
780 k_proj: alloc_f32(h, nk * hd)?,
781 v_proj: alloc_f32(h, nk * hd)?,
782 o_proj: alloc_f32(nh * hd, h)?,
783 gate_proj: alloc_f32(h, f)?,
784 up_proj: alloc_f32(h, f)?,
785 down_proj: alloc_f32(f, h)?,
786 norm1: alloc_f32_1d(h)?,
787 norm2: alloc_f32_1d(h)?,
788 });
789 }
790
791 Ok(Self {
792 embed_table,
793 layers,
794 final_norm,
795 lm_head,
796 lm_head_tied: tied,
797 })
798 }
799}
800
801fn rope_config_for_arch(arch: &str, info: &ModelInfo) -> Option<RopeConfig> {
808 let num_heads = info.num_heads.max(1) as usize;
809 let head_dim = info.hidden_dim as usize / num_heads;
810 let max_seq_len = info.context_len as usize;
811 match arch {
812 "llama3" => Some(RopeConfig {
813 head_dim,
814 base: 500_000.0,
815 max_seq_len,
816 }),
817 "llama" => Some(RopeConfig {
818 head_dim,
819 base: 10_000.0,
820 max_seq_len,
821 }),
822 "mistral" => Some(RopeConfig {
823 head_dim,
824 base: 10_000.0,
825 max_seq_len,
826 }),
827 "qwen2" | "qwen" => Some(RopeConfig {
828 head_dim,
829 base: 1_000_000.0,
830 max_seq_len,
831 }),
832 "phi3" => Some(RopeConfig {
833 head_dim,
834 base: 10_000.0,
835 max_seq_len,
836 }),
837 "gemma2" | "gemma" => Some(RopeConfig {
838 head_dim,
839 base: 10_000.0,
840 max_seq_len,
841 }),
842 _ => None,
843 }
844}
845
846pub trait LlmInference: Send + Sync {
857 fn load(model_path: &Path) -> TptrResult<Self>
858 where
859 Self: Sized;
860 fn model_info(&self) -> &ModelInfo;
861 fn infer(
862 &mut self,
863 tokens: &[u32],
864 max_new_tokens: u32,
865 callback: impl FnMut(u32) + Send,
866 ) -> TptrResult<()>;
867 fn cancel(&mut self);
868}
869
870pub struct GpuInferenceEngine {
879 info: ModelInfo,
880 template: ArchTemplate,
881 weights: ModelWeights,
882 embed_k: EmbeddingKernel,
883 rmsnorm_k: RmsNormKernel,
884 attn_k: AttentionKernel,
885 gemm_k: GemmKernel,
886 quant_gemm_k: QuantGemmKernel,
887 softmax_k: SoftmaxKernel,
888 kv_cache: KvCache,
889 cancelled: Arc<Mutex<bool>>,
890 rng_state: u64,
892 rope_config: Option<RopeConfig>,
895}
896
897impl std::fmt::Debug for GpuInferenceEngine {
898 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
899 f.debug_struct("GpuInferenceEngine")
900 .field("arch", &self.info.arch)
901 .field("num_layers", &self.info.num_layers)
902 .finish()
903 }
904}
905
906impl LlmInference for GpuInferenceEngine {
907 fn load(model_path: &Path) -> TptrResult<Self> {
908 let fmt = detect_format(model_path);
909 let info = match fmt {
910 ModelFormat::Gguf => parse_gguf_header(model_path)?,
911 ModelFormat::Tptf => parse_tptf_header(model_path)?,
912 ModelFormat::Unknown => {
913 parse_gguf_header(model_path)?
915 }
916 };
917 let template = template_for_arch(&info.arch, &info)?;
918
919 let tied = template
921 .post_ops
922 .iter()
923 .any(|op| matches!(op, ForwardOp::LinearOut { tied: true, .. }));
924
925 let weights = if fmt == ModelFormat::Tptf {
928 ModelWeights::load_tptf(model_path, &info, tied)?
929 } else {
930 ModelWeights::allocate(&info, tied)?
931 };
932
933 let vendor = VendorBackend::detect();
935 let embed_k = EmbeddingKernel::with_vendor(vendor.clone());
936 let rmsnorm_k = RmsNormKernel::with_vendor(vendor.clone());
937 let attn_k = AttentionKernel::with_vendor(vendor.clone());
938 let gemm_k = GemmKernel::with_vendor(vendor.clone());
939 let quant_gemm_k = QuantGemmKernel::with_vendor(vendor.clone());
940 let softmax_k = SoftmaxKernel::with_vendor(vendor);
941
942 let head_dim = info.hidden_dim / info.num_heads.max(1);
943 let kv_cache = KvCache::new(
944 info.num_layers,
945 info.context_len,
946 info.num_kv_heads.max(1),
947 head_dim,
948 );
949
950 let rope_config = rope_config_for_arch(&info.arch, &info);
951
952 Ok(Self {
953 info,
954 template,
955 weights,
956 embed_k,
957 rmsnorm_k,
958 attn_k,
959 gemm_k,
960 quant_gemm_k,
961 softmax_k,
962 kv_cache,
963 cancelled: Arc::new(Mutex::new(false)),
964 rng_state: 0x853c49e6748fea9b,
965 rope_config,
966 })
967 }
968
969 fn model_info(&self) -> &ModelInfo {
970 &self.info
971 }
972
973 fn infer(
974 &mut self,
975 tokens: &[u32],
976 max_new_tokens: u32,
977 mut callback: impl FnMut(u32) + Send,
978 ) -> TptrResult<()> {
979 if tokens.is_empty() {
980 return Err(TptrError::new(
981 ErrorCode::ArgumentMismatch,
982 "token list is empty",
983 ));
984 }
985
986 self.kv_cache.reset();
987
988 let mut current_token = *tokens.last().unwrap();
991
992 for _ in 0..max_new_tokens {
993 if *self.cancelled.lock().unwrap() {
994 break;
995 }
996
997 let next = self.forward_step(current_token)?;
998 callback(next);
999 current_token = next;
1000 }
1001
1002 Ok(())
1003 }
1004
1005 fn cancel(&mut self) {
1006 *self.cancelled.lock().unwrap() = true;
1007 }
1008}
1009
1010impl GpuInferenceEngine {
1011 fn forward_step(&mut self, token: u32) -> TptrResult<u32> {
1015 let hidden_dim = self.info.hidden_dim as usize;
1016 let vocab_size = self.info.vocab_size as usize;
1017 let num_heads = self.info.num_heads.max(1) as usize;
1018 let num_kv = self.info.num_kv_heads.max(1) as usize;
1019 let head_dim = hidden_dim / num_heads;
1020 let ffn_dim = self.info.ffn_dim as usize;
1021
1022 let idx_buf = token_to_index_buffer(token)?;
1024 let mut hidden = self
1025 .embed_k
1026 .execute(&self.weights.embed_table, &idx_buf)
1027 .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, e.to_string()))?;
1028 hidden = reshape_to_2d(hidden, 1, hidden_dim)?;
1030
1031 for layer_idx in 0..self.info.num_layers as usize {
1033 if *self.cancelled.lock().unwrap() {
1034 return Ok(0);
1035 }
1036
1037 let layer_bits = self
1039 .info
1040 .per_layer_bits
1041 .get(layer_idx)
1042 .copied()
1043 .unwrap_or(0);
1044
1045 let layer = &self.weights.layers[layer_idx];
1046
1047 let normed = self.rmsnorm_k.execute(&hidden, &layer.norm1).map_err(|e| {
1049 TptrError::new(
1050 ErrorCode::LaunchFailure,
1051 format!("rmsnorm1 layer {}: {}", layer_idx, e),
1052 )
1053 })?;
1054 let normed = reshape_to_2d(normed, 1, hidden_dim)?;
1055
1056 macro_rules! proj_gemm {
1058 ($input:expr, $weight:expr, $name:literal) => {
1059 if layer_bits > 0 {
1060 quant_layer_gemm(&self.quant_gemm_k, $input, $weight, layer_bits).map_err(
1061 |e| {
1062 TptrError::new(
1063 ErrorCode::LaunchFailure,
1064 format!("{} layer {}: {}", $name, layer_idx, e),
1065 )
1066 },
1067 )
1068 } else {
1069 self.gemm_k
1070 .execute($input, $weight, None, 1.0, 0.0)
1071 .map_err(|e| {
1072 TptrError::new(
1073 ErrorCode::LaunchFailure,
1074 format!("{} layer {}: {}", $name, layer_idx, e),
1075 )
1076 })
1077 }
1078 };
1079 }
1080
1081 let q = proj_gemm!(&normed, &layer.q_proj, "q_proj")?;
1082 let k = proj_gemm!(&normed, &layer.k_proj, "k_proj")?;
1083 let v = proj_gemm!(&normed, &layer.v_proj, "v_proj")?;
1084
1085 let rope_pos = self.kv_cache.seq_len;
1089 let (q, k) = if let Some(ref rope_cfg) = self.rope_config {
1090 let mut q_data = buf_to_vec(&q);
1091 let mut k_data = buf_to_vec(&k);
1092 crate::rope::apply_rope(&mut q_data, &mut k_data, rope_pos, rope_cfg);
1093 let q_rotated = vec_to_buf(q_data, 1, num_heads * head_dim)?;
1094 let k_rotated = vec_to_buf(k_data, 1, num_kv * head_dim)?;
1095 (q_rotated, k_rotated)
1096 } else {
1097 (q, k)
1098 };
1099
1100 let kv_step = self.kv_cache.seq_len + 1;
1106 let k_data = buf_to_vec(&k);
1107 let v_data = buf_to_vec(&v);
1108 self.kv_cache.append(layer_idx, &k_data, &v_data);
1109
1110 let qk_dim = num_heads * head_dim;
1111 let kv_row = num_kv * head_dim;
1112 let k_full = expand_kv_heads(
1114 self.kv_cache.get_k_at(layer_idx, kv_step),
1115 kv_step,
1116 kv_row,
1117 qk_dim,
1118 );
1119 let v_full = expand_kv_heads(
1120 self.kv_cache.get_v_at(layer_idx, kv_step),
1121 kv_step,
1122 kv_row,
1123 qk_dim,
1124 );
1125 let k_cache = vec_to_buf(k_full, kv_step, qk_dim)?;
1126 let v_cache = vec_to_buf(v_full, kv_step, qk_dim)?;
1127
1128 let scale = Some(1.0 / (head_dim as f32).sqrt());
1129 let q2d = reshape_to_2d(q, 1, qk_dim)?;
1133 let attn_out = self
1134 .attn_k
1135 .execute(&q2d, &k_cache, &v_cache, scale, None)
1136 .map_err(|e| {
1137 TptrError::new(
1138 ErrorCode::LaunchFailure,
1139 format!("attention layer {}: {}", layer_idx, e),
1140 )
1141 })?;
1142 let attn_dim = attn_out_dim(&attn_out);
1143 let attn_out = reshape_to_2d(attn_out, 1, attn_dim)?;
1144
1145 let o = proj_gemm!(&attn_out, &layer.o_proj, "o_proj")?;
1147 let o = reshape_to_2d(o, 1, hidden_dim)?;
1148 hidden = elementwise_add_2d(&hidden, &o, hidden_dim)?;
1149
1150 let normed2 = self.rmsnorm_k.execute(&hidden, &layer.norm2).map_err(|e| {
1152 TptrError::new(
1153 ErrorCode::LaunchFailure,
1154 format!("rmsnorm2 layer {}: {}", layer_idx, e),
1155 )
1156 })?;
1157 let normed2 = reshape_to_2d(normed2, 1, hidden_dim)?;
1158
1159 let gate = proj_gemm!(&normed2, &layer.gate_proj, "gate_proj")?;
1161 let up = proj_gemm!(&normed2, &layer.up_proj, "up_proj")?;
1162
1163 let ffn_mid = swiglu_host(&gate, &up, ffn_dim)?;
1164 let ffn_mid = reshape_to_2d(ffn_mid, 1, ffn_dim)?;
1165
1166 let ffn_out = proj_gemm!(&ffn_mid, &layer.down_proj, "down_proj")?;
1167 let ffn_out = reshape_to_2d(ffn_out, 1, hidden_dim)?;
1168
1169 hidden = elementwise_add_2d(&hidden, &ffn_out, hidden_dim)?;
1170 }
1171
1172 let normed_final = self
1174 .rmsnorm_k
1175 .execute(&hidden, &self.weights.final_norm)
1176 .map_err(|e| {
1177 TptrError::new(ErrorCode::LaunchFailure, format!("final rmsnorm: {}", e))
1178 })?;
1179 let normed_final = reshape_to_2d(normed_final, 1, hidden_dim)?;
1180
1181 let lm_weight = if self.weights.lm_head_tied {
1182 &self.weights.embed_table
1183 } else {
1184 &self.weights.lm_head
1185 };
1186 let logits_raw = if self.weights.lm_head_tied {
1191 tied_lm_head_host(
1193 &normed_final,
1194 &self.weights.embed_table,
1195 hidden_dim,
1196 vocab_size,
1197 )?
1198 } else {
1199 self.gemm_k
1200 .execute(&normed_final, lm_weight, None, 1.0, 0.0)
1201 .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, format!("lm_head: {}", e)))?
1202 };
1203
1204 let logits = reshape_to_2d(logits_raw, 1, vocab_size)?;
1205
1206 let (temperature, top_k) = self
1208 .template
1209 .post_ops
1210 .iter()
1211 .find_map(|op| {
1212 if let ForwardOp::Sampling { temperature, top_k } = op {
1213 Some((*temperature, *top_k))
1214 } else {
1215 None
1216 }
1217 })
1218 .unwrap_or((1.0, 1));
1219
1220 if temperature <= 0.0 || top_k <= 1 {
1221 let probs = self
1223 .softmax_k
1224 .execute(&logits)
1225 .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, format!("softmax: {}", e)))?;
1226 return Ok(sample_argmax(&probs, vocab_size));
1227 }
1228
1229 let next = self.sample_with_temperature(&logits, vocab_size, temperature, top_k)?;
1231 Ok(next)
1232 }
1233
1234 fn sample_with_temperature(
1240 &mut self,
1241 logits: &GpuBuffer<f32>,
1242 vocab_size: usize,
1243 temperature: f32,
1244 top_k: u32,
1245 ) -> TptrResult<u32> {
1246 let mut raw = vec![0.0f32; vocab_size];
1247 let n = raw.len().min(logits.num_elements());
1248 let _ = logits.copy_to_host(&mut raw[..n]);
1249
1250 for x in raw.iter_mut() {
1252 *x /= temperature;
1253 }
1254
1255 let k = (top_k as usize).min(vocab_size).max(1);
1257 let mut indexed: Vec<(usize, f32)> = raw.iter().cloned().enumerate().collect();
1258 indexed.sort_unstable_by(|(_, a), (_, b)| {
1259 b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
1260 });
1261 indexed.truncate(k);
1262
1263 let max_val = indexed
1265 .iter()
1266 .map(|(_, v)| *v)
1267 .fold(f32::NEG_INFINITY, f32::max);
1268 let mut expd: Vec<f32> = indexed.iter().map(|(_, v)| (*v - max_val).exp()).collect();
1269 let sum: f32 = expd.iter().sum();
1270 if sum > 0.0 {
1271 for e in expd.iter_mut() {
1272 *e /= sum;
1273 }
1274 }
1275
1276 let u = xorshift64(&mut self.rng_state);
1278 let mut cumsum = 0.0f32;
1279 for (i, p) in expd.iter().enumerate() {
1280 cumsum += p;
1281 if u <= cumsum || i + 1 == k {
1282 return Ok(indexed[i].0 as u32);
1283 }
1284 }
1285 Ok(indexed[0].0 as u32)
1286 }
1287}
1288
1289fn token_to_index_buffer(token: u32) -> TptrResult<GpuBuffer<i32>> {
1295 let mut buf = GpuBuffer::<i32>::new(Shape::new(&[1]), DType::I32, BufferFlags::STORAGE)
1296 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1297 buf.copy_from_host(&[token as i32])
1298 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1299 Ok(buf)
1300}
1301
1302fn reshape_to_2d(buf: GpuBuffer<f32>, rows: usize, cols: usize) -> TptrResult<GpuBuffer<f32>> {
1305 let n = rows * cols;
1306 let mut src = vec![0.0f32; buf.num_elements().max(1)];
1307 let copy_len = src.len().min(buf.num_elements());
1308 let _ = buf.copy_to_host(&mut src[..copy_len]);
1309 let mut out = GpuBuffer::<f32>::new(Shape::dim2(rows, cols), DType::F32, BufferFlags::STORAGE)
1310 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1311 let mut dst = vec![0.0f32; n];
1312 let fill_len = dst.len().min(src.len());
1313 dst[..fill_len].copy_from_slice(&src[..fill_len]);
1314 out.copy_from_host(&dst)
1315 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1316 Ok(out)
1317}
1318
1319fn attn_out_dim(buf: &GpuBuffer<f32>) -> usize {
1321 buf.dim(1).unwrap_or(buf.num_elements())
1322}
1323
1324fn quant_layer_gemm(
1333 quant_k: &QuantGemmKernel,
1334 activation: &GpuBuffer<f32>,
1335 weight: &GpuBuffer<f32>,
1336 bits: u8,
1337) -> TptrResult<GpuBuffer<f32>> {
1338 let in_dim = weight.dim(0).unwrap_or(0);
1339 let out_dim = weight.dim(1).unwrap_or(0);
1340 if in_dim == 0 || out_dim == 0 {
1341 return Err(TptrError::new(
1342 ErrorCode::ArgumentMismatch,
1343 "quant_layer_gemm: weight has zero dimension",
1344 ));
1345 }
1346
1347 let mut w = vec![0.0f32; in_dim * out_dim];
1349 let wn = w.len().min(weight.num_elements());
1350 let _ = weight.copy_to_host(&mut w[..wn]);
1351
1352 let mut act = vec![0.0f32; in_dim];
1353 let an = act.len().min(activation.num_elements());
1354 let _ = activation.copy_to_host(&mut act[..an]);
1355
1356 let mut wt = vec![0.0f32; out_dim * in_dim];
1358 for r in 0..in_dim {
1359 for c in 0..out_dim {
1360 wt[c * in_dim + r] = w[r * out_dim + c];
1361 }
1362 }
1363
1364 let bits_u = bits.max(1) as usize;
1366 let pack_factor = (8usize / bits_u).max(1);
1367 let packed_k = in_dim.div_ceil(pack_factor);
1368 let group_size = tpt_gpu_primitives::DEFAULT_GROUP_SIZE as usize;
1369 let groups_per_row = in_dim.div_ceil(group_size);
1370 let max_int = ((1u32 << bits) - 1) as f32 / 2.0;
1371
1372 let mut scales = vec![1.0f32; out_dim * groups_per_row];
1373 for row in 0..out_dim {
1374 for g in 0..groups_per_row {
1375 let start = row * in_dim + g * group_size;
1376 let end = (start + group_size).min(row * in_dim + in_dim);
1377 let max_abs = wt[start..end]
1378 .iter()
1379 .map(|x| x.abs())
1380 .fold(0.0f32, f32::max);
1381 scales[row * groups_per_row + g] = if max_abs > 0.0 {
1382 max_abs / max_int
1383 } else {
1384 1.0
1385 };
1386 }
1387 }
1388
1389 let mut packed = vec![0i8; out_dim * packed_k];
1391 for row in 0..out_dim {
1392 for col in 0..in_dim {
1393 let g = col / group_size;
1394 let scale = scales[row * groups_per_row + g];
1395 let val = (wt[row * in_dim + col] / scale)
1396 .round()
1397 .clamp(-max_int, max_int) as i8;
1398 let packed_col = col / pack_factor;
1399 let shift = (col % pack_factor) * bits_u;
1400 packed[row * packed_k + packed_col] |= ((val as u8) << shift) as i8;
1401 }
1402 }
1403
1404 let mut a_buf = GpuBuffer::<i8>::new(
1406 Shape::dim2(out_dim, packed_k),
1407 DType::I8,
1408 BufferFlags::STORAGE,
1409 )
1410 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1411 a_buf
1412 .copy_from_host(&packed)
1413 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1414
1415 let mut b_buf = GpuBuffer::<f32>::new(Shape::dim2(in_dim, 1), DType::F32, BufferFlags::STORAGE)
1417 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1418 b_buf
1419 .copy_from_host(&act)
1420 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1421
1422 let mut sc_buf = GpuBuffer::<f32>::new(
1423 Shape::dim2(out_dim, groups_per_row),
1424 DType::F32,
1425 BufferFlags::STORAGE,
1426 )
1427 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1428 sc_buf
1429 .copy_from_host(&scales)
1430 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1431
1432 let zp_buf = GpuBuffer::<i8>::new(
1433 Shape::dim2(out_dim, groups_per_row),
1434 DType::I8,
1435 BufferFlags::STORAGE,
1436 )
1437 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1438
1439 let result = quant_k
1441 .execute(&a_buf, &b_buf, &sc_buf, &zp_buf)
1442 .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, e.to_string()))?;
1443
1444 let mut out_raw = vec![0.0f32; out_dim];
1446 let rn = out_raw.len().min(result.num_elements());
1447 let _ = result.copy_to_host(&mut out_raw[..rn]);
1448 vec_to_buf(out_raw, 1, out_dim)
1449}
1450
1451fn expand_kv_heads(src: &[f32], seq_len: usize, kv_row: usize, qk_dim: usize) -> Vec<f32> {
1458 if seq_len == 0 || qk_dim == 0 {
1459 return vec![];
1460 }
1461 if kv_row >= qk_dim {
1462 let mut out = vec![0.0f32; seq_len * qk_dim];
1464 for t in 0..seq_len {
1465 let src_start = t * kv_row;
1466 let dst_start = t * qk_dim;
1467 let copy = qk_dim.min(kv_row).min(src.len().saturating_sub(src_start));
1468 if copy > 0 {
1469 out[dst_start..dst_start + copy].copy_from_slice(&src[src_start..src_start + copy]);
1470 }
1471 }
1472 return out;
1473 }
1474 let mut out = vec![0.0f32; seq_len * qk_dim];
1476 for t in 0..seq_len {
1477 let src_start = t * kv_row;
1478 let dst_start = t * qk_dim;
1479 for i in 0..qk_dim {
1480 let src_i = src_start + (i % kv_row);
1481 if src_i < src.len() {
1482 out[dst_start + i] = src[src_i];
1483 }
1484 }
1485 }
1486 out
1487}
1488
1489fn tptf_dequantize(
1498 quantized: &[u8],
1499 scales: &[f32],
1500 zero_points: &[i8],
1501 bits: u8,
1502 group_size: usize,
1503 num_elements: usize,
1504) -> Vec<f32> {
1505 if bits >= 16 {
1506 let float_count = num_elements.min(quantized.len() / 4);
1508 let mut result = vec![0.0f32; num_elements];
1509 for i in 0..float_count {
1510 let b = &quantized[i * 4..(i + 1) * 4];
1511 result[i] = f32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1512 }
1513 return result;
1514 }
1515
1516 let pack_factor = (8usize / bits as usize).max(1);
1517 let mask = (1u16 << bits) - 1;
1518 let mut result = vec![0.0f32; num_elements];
1519
1520 for (elem_idx, out) in result.iter_mut().enumerate() {
1521 let packed_byte_idx = elem_idx / pack_factor;
1522 let bit_offset = (elem_idx % pack_factor) * bits as usize;
1523
1524 if packed_byte_idx < quantized.len() {
1525 let byte = quantized[packed_byte_idx] as u16;
1526 let q = (byte >> bit_offset) & mask;
1527
1528 let group_idx = elem_idx / group_size;
1529 let scale = scales.get(group_idx).copied().unwrap_or(1.0);
1530 let zp = zero_points.get(group_idx).copied().unwrap_or(0) as f32;
1531
1532 *out = (q as f32 + zp) * scale;
1533 }
1534 }
1535
1536 result
1537}
1538
1539pub fn execute_sampling(logits: &[f32], temperature: f32, top_k: usize) -> usize {
1546 if logits.is_empty() {
1547 return 0;
1548 }
1549
1550 if temperature == 0.0 || top_k <= 1 {
1552 return logits
1553 .iter()
1554 .enumerate()
1555 .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
1556 .map(|(i, _)| i)
1557 .unwrap_or(0);
1558 }
1559
1560 let mut scaled: Vec<(usize, f32)> = logits
1562 .iter()
1563 .enumerate()
1564 .map(|(i, &v)| (i, v / temperature))
1565 .collect();
1566
1567 let k = top_k.min(logits.len()).max(1);
1569 scaled.sort_unstable_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
1570 scaled.truncate(k);
1571
1572 let max_val = scaled
1574 .iter()
1575 .map(|(_, v)| *v)
1576 .fold(f32::NEG_INFINITY, f32::max);
1577 let mut expd: Vec<f32> = scaled.iter().map(|(_, v)| (*v - max_val).exp()).collect();
1578 let sum: f32 = expd.iter().sum();
1579 if sum > 0.0 {
1580 for e in expd.iter_mut() {
1581 *e /= sum;
1582 }
1583 }
1584
1585 use std::sync::atomic::{AtomicU64, Ordering};
1587 static PRNG: AtomicU64 = AtomicU64::new(0x853c49e6748fea9b);
1588 let u = {
1589 let mut x = PRNG.load(Ordering::Relaxed);
1590 x ^= x << 13;
1591 x ^= x >> 7;
1592 x ^= x << 17;
1593 PRNG.store(x, Ordering::Relaxed);
1594 (x as f64 / u64::MAX as f64) as f32
1595 };
1596
1597 let mut cumsum = 0.0f32;
1598 for (i, p) in expd.iter().enumerate() {
1599 cumsum += p;
1600 if u <= cumsum || i + 1 == k {
1601 return scaled[i].0;
1602 }
1603 }
1604 scaled[0].0
1605}
1606
1607fn xorshift64(state: &mut u64) -> f32 {
1609 let mut x = *state;
1610 x ^= x << 13;
1611 x ^= x >> 7;
1612 x ^= x << 17;
1613 *state = x;
1614 (x as f64 / u64::MAX as f64) as f32
1616}
1617
1618#[allow(dead_code)]
1621fn pad_to_len(src: &[f32], target_len: usize) -> Vec<f32> {
1622 let mut out = vec![0.0f32; target_len];
1623 let copy = src.len().min(target_len);
1624 out[..copy].copy_from_slice(&src[..copy]);
1625 out
1626}
1627
1628fn buf_to_vec(buf: &GpuBuffer<f32>) -> Vec<f32> {
1630 let n = buf.num_elements();
1631 let mut v = vec![0.0f32; n];
1632 let _ = buf.copy_to_host(&mut v);
1633 v
1634}
1635
1636fn vec_to_buf(data: Vec<f32>, rows: usize, cols: usize) -> TptrResult<GpuBuffer<f32>> {
1638 let mut buf = GpuBuffer::<f32>::new(Shape::dim2(rows, cols), DType::F32, BufferFlags::STORAGE)
1639 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1640 let n = (rows * cols).min(data.len());
1641 let mut padded = vec![0.0f32; rows * cols];
1642 padded[..n].copy_from_slice(&data[..n]);
1643 buf.copy_from_host(&padded)
1644 .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1645 Ok(buf)
1646}
1647
1648fn elementwise_add_2d(
1650 a: &GpuBuffer<f32>,
1651 b: &GpuBuffer<f32>,
1652 dim: usize,
1653) -> TptrResult<GpuBuffer<f32>> {
1654 let mut va = vec![0.0f32; dim];
1655 let mut vb = vec![0.0f32; dim];
1656 let _ = a.copy_to_host(&mut va);
1657 let bn = vb.len().min(b.num_elements());
1658 let _ = b.copy_to_host(&mut vb[..bn]);
1659 for (x, y) in va.iter_mut().zip(vb.iter()) {
1660 *x += y;
1661 }
1662 vec_to_buf(va, 1, dim)
1663}
1664
1665fn swiglu_host(
1668 gate: &GpuBuffer<f32>,
1669 up: &GpuBuffer<f32>,
1670 ffn_dim: usize,
1671) -> TptrResult<GpuBuffer<f32>> {
1672 let mut g = vec![0.0f32; ffn_dim];
1673 let mut u = vec![0.0f32; ffn_dim];
1674 let gn = g.len().min(gate.num_elements());
1675 let _ = gate.copy_to_host(&mut g[..gn]);
1676 let un = u.len().min(up.num_elements());
1677 let _ = up.copy_to_host(&mut u[..un]);
1678 for (gi, ui) in g.iter_mut().zip(u.iter()) {
1679 let silu = *gi / (1.0 + (-*gi).exp()); *gi = silu * ui;
1681 }
1682 vec_to_buf(g, 1, ffn_dim)
1683}
1684
1685fn tied_lm_head_host(
1689 hidden: &GpuBuffer<f32>,
1690 embed_table: &GpuBuffer<f32>,
1691 hidden_dim: usize,
1692 vocab_size: usize,
1693) -> TptrResult<GpuBuffer<f32>> {
1694 let mut h = vec![0.0f32; hidden_dim];
1695 let mut e = vec![0.0f32; vocab_size * hidden_dim];
1696 let hn = h.len().min(hidden.num_elements());
1697 let _ = hidden.copy_to_host(&mut h[..hn]);
1698 let en = e.len().min(embed_table.num_elements());
1699 let _ = embed_table.copy_to_host(&mut e[..en]);
1700
1701 let mut logits = vec![0.0f32; vocab_size];
1702 for v in 0..vocab_size {
1703 let row = &e[v * hidden_dim..(v + 1) * hidden_dim];
1704 logits[v] = h.iter().zip(row.iter()).map(|(a, b)| a * b).sum();
1705 }
1706 vec_to_buf(logits, 1, vocab_size)
1707}
1708
1709fn sample_argmax(probs: &GpuBuffer<f32>, vocab_size: usize) -> u32 {
1711 let mut p = vec![0.0f32; vocab_size];
1712 let pn = p.len().min(probs.num_elements());
1713 let _ = probs.copy_to_host(&mut p[..pn]);
1714 p.iter()
1715 .enumerate()
1716 .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
1717 .map(|(i, _)| i as u32)
1718 .unwrap_or(0)
1719}
1720
1721#[cfg(test)]
1726mod tests {
1727 use super::*;
1728 use std::fs;
1729
1730 fn write_gguf(path: &Path, kv: &[(&str, GgufKvVal)]) {
1732 let mut buf: Vec<u8> = Vec::new();
1733 buf.extend_from_slice(b"GGUF"); buf.extend_from_slice(&2u32.to_le_bytes()); buf.extend_from_slice(&0u64.to_le_bytes()); buf.extend_from_slice(&(kv.len() as u64).to_le_bytes()); for (key, val) in kv {
1738 let kb = key.as_bytes();
1739 buf.extend_from_slice(&(kb.len() as u64).to_le_bytes());
1740 buf.extend_from_slice(kb);
1741 match val {
1742 GgufKvVal::Str(s) => {
1743 buf.extend_from_slice(&8u32.to_le_bytes()); let sb = s.as_bytes();
1745 buf.extend_from_slice(&(sb.len() as u64).to_le_bytes());
1746 buf.extend_from_slice(sb);
1747 }
1748 GgufKvVal::U32(v) => {
1749 buf.extend_from_slice(&4u32.to_le_bytes()); buf.extend_from_slice(&v.to_le_bytes());
1751 }
1752 }
1753 }
1754 fs::write(path, &buf).unwrap();
1755 }
1756
1757 enum GgufKvVal {
1758 Str(String),
1759 U32(u32),
1760 }
1761
1762 fn tmp(name: &str) -> PathBuf {
1763 let mut p = std::env::temp_dir();
1764 p.push(format!("tptr_infer_{}_{}", std::process::id(), name));
1765 p
1766 }
1767
1768 fn write_tptf(path: &Path, arch: &str, num_layers: u32, per_layer_bits: &[u8]) {
1773 let mut buf = vec![0u8; 512];
1774 buf[0..4].copy_from_slice(b"TPTF");
1775 buf[4..8].copy_from_slice(&1u32.to_le_bytes()); buf[8..12].copy_from_slice(&0u32.to_le_bytes()); let ab = arch.as_bytes();
1778 buf[12] = ab.len().min(63) as u8;
1779 buf[13..13 + ab.len().min(63)].copy_from_slice(&ab[..ab.len().min(63)]);
1780 buf[76..80].copy_from_slice(&64u32.to_le_bytes()); buf[80..84].copy_from_slice(&32000u32.to_le_bytes()); buf[84..88].copy_from_slice(&64u32.to_le_bytes()); buf[88..92].copy_from_slice(&4u32.to_le_bytes()); buf[92..96].copy_from_slice(&2u32.to_le_bytes()); buf[96..100].copy_from_slice(&128u32.to_le_bytes()); buf[100..104].copy_from_slice(&num_layers.to_le_bytes());
1787 for (i, &b) in per_layer_bits.iter().enumerate().take(128) {
1788 buf[104 + i] = b;
1789 }
1790 fs::write(path, &buf).unwrap();
1791 }
1792
1793 #[test]
1794 fn parse_tptf_header_basic() {
1795 let p = tmp("basic.tptf");
1796 write_tptf(&p, "llama3", 4, &[4, 4, 6, 8]);
1797 let info = parse_tptf_header(&p).unwrap();
1798 assert_eq!(info.arch, "llama3");
1799 assert_eq!(info.num_layers, 4);
1800 assert_eq!(info.per_layer_bits, vec![4, 4, 6, 8]);
1801 let _ = fs::remove_file(&p);
1802 }
1803
1804 #[test]
1805 fn detect_format_tptf() {
1806 let p = tmp("detect.tptf");
1807 write_tptf(&p, "llama3", 2, &[]);
1808 assert_eq!(detect_format(&p), ModelFormat::Tptf);
1809 let _ = fs::remove_file(&p);
1810 }
1811
1812 #[test]
1813 fn load_tptf_via_engine() {
1814 let p = tmp("engine.tptf");
1815 write_tptf(&p, "llama3", 2, &[4, 4]);
1816 let engine = GpuInferenceEngine::load(&p).unwrap();
1817 assert_eq!(engine.model_info().arch, "llama3");
1818 assert_eq!(engine.model_info().per_layer_bits, vec![4, 4]);
1819 let _ = fs::remove_file(&p);
1820 }
1821
1822 #[test]
1823 fn parse_bad_magic_errors() {
1824 let p = tmp("bad_magic.gguf");
1825 fs::write(
1826 &p,
1827 b"NOTG\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
1828 )
1829 .unwrap();
1830 let err = parse_gguf_header(&p).unwrap_err();
1831 assert_eq!(err.code, ErrorCode::InvalidKernel);
1832 let _ = fs::remove_file(&p);
1833 }
1834
1835 #[test]
1836 fn parse_missing_file_errors() {
1837 let err = parse_gguf_header(Path::new("/nonexistent/model.gguf")).unwrap_err();
1838 assert_eq!(err.code, ErrorCode::InvalidKernel);
1839 }
1840
1841 #[test]
1842 fn parse_extracts_arch_and_dims() {
1843 let p = tmp("parse_arch.gguf");
1844 write_gguf(
1845 &p,
1846 &[
1847 ("general.architecture", GgufKvVal::Str("llama3".into())),
1848 ("llm.context_length", GgufKvVal::U32(8192)),
1849 ("llm.embedding_length", GgufKvVal::U32(4096)),
1850 ("llm.attention.head_count", GgufKvVal::U32(32)),
1851 ("llm.attention.head_count_kv", GgufKvVal::U32(8)),
1852 ("llm.feed_forward_length", GgufKvVal::U32(11008)),
1853 ("llm.block_count", GgufKvVal::U32(32)),
1854 ],
1855 );
1856 let info = parse_gguf_header(&p).unwrap();
1857 assert_eq!(info.arch, "llama3");
1858 assert_eq!(info.context_len, 8192);
1859 assert_eq!(info.hidden_dim, 4096);
1860 assert_eq!(info.num_heads, 32);
1861 assert_eq!(info.num_kv_heads, 8);
1862 assert_eq!(info.num_layers, 32);
1863 let _ = fs::remove_file(&p);
1864 }
1865
1866 #[test]
1867 fn parse_qwen2_arch() {
1868 let p = tmp("qwen2.gguf");
1869 write_gguf(
1870 &p,
1871 &[
1872 ("general.architecture", GgufKvVal::Str("qwen2".into())),
1873 ("llm.context_length", GgufKvVal::U32(32768)),
1874 ("llm.embedding_length", GgufKvVal::U32(3584)),
1875 ("llm.attention.head_count", GgufKvVal::U32(28)),
1876 ("llm.attention.head_count_kv", GgufKvVal::U32(4)),
1877 ("llm.feed_forward_length", GgufKvVal::U32(18944)),
1878 ("llm.block_count", GgufKvVal::U32(28)),
1879 ],
1880 );
1881 let info = parse_gguf_header(&p).unwrap();
1882 assert_eq!(info.arch, "qwen2");
1883 assert_eq!(info.hidden_dim, 3584);
1884 assert_eq!(info.num_kv_heads, 4);
1885 let _ = fs::remove_file(&p);
1886 }
1887
1888 fn write_minimal_gguf(path: &Path, arch: &str) {
1891 write_gguf(
1892 path,
1893 &[
1894 ("general.architecture", GgufKvVal::Str(arch.into())),
1895 ("llm.context_length", GgufKvVal::U32(64)),
1896 ("llm.embedding_length", GgufKvVal::U32(64)),
1897 ("llm.attention.head_count", GgufKvVal::U32(4)),
1898 ("llm.attention.head_count_kv", GgufKvVal::U32(2)),
1899 ("llm.feed_forward_length", GgufKvVal::U32(128)),
1900 ("llm.block_count", GgufKvVal::U32(2)),
1901 ],
1902 );
1903 }
1904
1905 #[test]
1906 fn load_llama3_succeeds() {
1907 let p = tmp("llama3_load.gguf");
1908 write_minimal_gguf(&p, "llama3");
1909 let engine = GpuInferenceEngine::load(&p).unwrap();
1910 assert_eq!(engine.model_info().arch, "llama3");
1911 let _ = fs::remove_file(&p);
1912 }
1913
1914 #[test]
1915 fn load_qwen2_succeeds() {
1916 let p = tmp("qwen2_load.gguf");
1917 write_minimal_gguf(&p, "qwen2");
1918 let engine = GpuInferenceEngine::load(&p).unwrap();
1919 assert_eq!(engine.model_info().arch, "qwen2");
1920 let _ = fs::remove_file(&p);
1921 }
1922
1923 #[test]
1924 fn load_unknown_arch_errors() {
1925 let p = tmp("gpt2_load.gguf");
1926 write_minimal_gguf(&p, "gpt2");
1927 let err = GpuInferenceEngine::load(&p).unwrap_err();
1928 assert_eq!(err.code, ErrorCode::UnsupportedFeature);
1929 let _ = fs::remove_file(&p);
1930 }
1931
1932 #[test]
1933 fn infer_produces_max_new_tokens() {
1934 let p = tmp("infer_count.gguf");
1935 write_minimal_gguf(&p, "llama3");
1936 let mut engine = GpuInferenceEngine::load(&p).unwrap();
1937 let mut generated = Vec::new();
1938 engine
1939 .infer(&[1, 2, 3], 4, |tok| generated.push(tok))
1940 .unwrap();
1941 assert_eq!(generated.len(), 4);
1942 let _ = fs::remove_file(&p);
1943 }
1944
1945 #[test]
1946 fn infer_empty_tokens_errors() {
1947 let p = tmp("infer_empty.gguf");
1948 write_minimal_gguf(&p, "llama3");
1949 let mut engine = GpuInferenceEngine::load(&p).unwrap();
1950 let err = engine.infer(&[], 4, |_| {}).unwrap_err();
1951 assert_eq!(err.code, ErrorCode::ArgumentMismatch);
1952 let _ = fs::remove_file(&p);
1953 }
1954
1955 #[test]
1956 fn cancel_stops_generation() {
1957 let p = tmp("cancel_gen.gguf");
1958 write_minimal_gguf(&p, "llama3");
1959 let mut engine = GpuInferenceEngine::load(&p).unwrap();
1960 engine.cancel();
1961 let mut generated = Vec::new();
1962 engine
1963 .infer(&[1, 2, 3], 100, |tok| generated.push(tok))
1964 .unwrap();
1965 assert!(
1966 generated.is_empty(),
1967 "cancel before infer should produce no tokens"
1968 );
1969 let _ = fs::remove_file(&p);
1970 }
1971
1972 #[test]
1973 fn infer_gemma2_succeeds() {
1974 let p = tmp("gemma2_infer.gguf");
1975 write_minimal_gguf(&p, "gemma2");
1976 let mut engine = GpuInferenceEngine::load(&p).unwrap();
1977 let mut generated = Vec::new();
1978 engine.infer(&[1], 2, |tok| generated.push(tok)).unwrap();
1979 assert_eq!(generated.len(), 2);
1980 let _ = fs::remove_file(&p);
1981 }
1982
1983 #[allow(clippy::too_many_arguments)]
1987 fn append_tptf_tensor_block(
1988 data: &mut Vec<u8>,
1989 layer_idx: u32,
1990 name: &str,
1991 bits: u8,
1992 group_size: u32,
1993 rows: u32,
1994 cols: u32,
1995 weight_bytes: &[u8],
1996 ) {
1997 data.extend_from_slice(&layer_idx.to_le_bytes());
1998 let nb = name.as_bytes();
1999 data.push(nb.len().min(31) as u8);
2000 data.extend_from_slice(&nb[..nb.len().min(31)]);
2001 data.push(bits);
2002 data.extend_from_slice(&group_size.to_le_bytes());
2003 data.extend_from_slice(&rows.to_le_bytes());
2004 data.extend_from_slice(&cols.to_le_bytes());
2005 data.extend_from_slice(&(weight_bytes.len() as u32).to_le_bytes());
2007 data.extend_from_slice(weight_bytes);
2008 data.extend_from_slice(&1u32.to_le_bytes());
2010 data.extend_from_slice(&1.0f32.to_le_bytes());
2011 data.extend_from_slice(&1u32.to_le_bytes());
2013 data.push(0u8);
2014 while !data.len().is_multiple_of(128) {
2016 data.push(0u8);
2017 }
2018 }
2019
2020 fn write_tptf_with_tensors(path: &Path) {
2023 let mut buf = vec![0u8; 512];
2025 buf[0..4].copy_from_slice(b"TPTF");
2026 buf[4..8].copy_from_slice(&1u32.to_le_bytes()); buf[8..12].copy_from_slice(&0u32.to_le_bytes()); let ab = b"llama3";
2029 buf[12] = ab.len() as u8;
2030 buf[13..13 + ab.len()].copy_from_slice(ab);
2031 buf[76..80].copy_from_slice(&64u32.to_le_bytes()); buf[80..84].copy_from_slice(&8u32.to_le_bytes()); buf[84..88].copy_from_slice(&4u32.to_le_bytes()); buf[88..92].copy_from_slice(&2u32.to_le_bytes()); buf[92..96].copy_from_slice(&1u32.to_le_bytes()); buf[96..100].copy_from_slice(&8u32.to_le_bytes()); buf[100..104].copy_from_slice(&1u32.to_le_bytes()); buf[460..468].copy_from_slice(&512u64.to_le_bytes());
2040 let mut data = buf;
2043
2044 let gate_vals: Vec<f32> = (1u32..=32).map(|i| i as f32 * 0.1).collect();
2046 let gate_bytes: Vec<u8> = gate_vals.iter().flat_map(|v| v.to_le_bytes()).collect();
2047 append_tptf_tensor_block(&mut data, 0, "gate_proj", 16, 32, 4, 8, &gate_bytes);
2048
2049 let up_vals: Vec<f32> = (1u32..=32).map(|i| i as f32 * 0.2).collect();
2051 let up_bytes: Vec<u8> = up_vals.iter().flat_map(|v| v.to_le_bytes()).collect();
2052 append_tptf_tensor_block(&mut data, 0, "up_proj", 16, 32, 4, 8, &up_bytes);
2053
2054 fs::write(path, &data).unwrap();
2055 }
2056
2057 #[test]
2058 fn test_load_tptf_roundtrip() {
2059 let p = tmp("tptf_roundtrip.tptf");
2060 write_tptf_with_tensors(&p);
2061
2062 let info = parse_tptf_header(&p).unwrap();
2063 assert_eq!(info.arch, "llama3");
2064 assert_eq!(info.num_layers, 1);
2065 assert_eq!(info.hidden_dim, 4);
2066 assert_eq!(info.ffn_dim, 8);
2067
2068 let weights = ModelWeights::load_tptf(&p, &info, false).unwrap();
2070
2071 let gate = buf_to_vec(&weights.layers[0].gate_proj);
2073 assert_eq!(gate.len(), 4 * 8, "gate_proj shape mismatch");
2074 for (i, &v) in gate.iter().enumerate() {
2075 let expected = (i + 1) as f32 * 0.1;
2076 assert!(
2077 (v - expected).abs() < 1e-5,
2078 "gate_proj[{}]: expected {:.4}, got {:.4}",
2079 i,
2080 expected,
2081 v
2082 );
2083 }
2084
2085 let up = buf_to_vec(&weights.layers[0].up_proj);
2087 assert_eq!(up.len(), 4 * 8, "up_proj shape mismatch");
2088 for (i, &v) in up.iter().enumerate() {
2089 let expected = (i + 1) as f32 * 0.2;
2090 assert!(
2091 (v - expected).abs() < 1e-5,
2092 "up_proj[{}]: expected {:.4}, got {:.4}",
2093 i,
2094 expected,
2095 v
2096 );
2097 }
2098
2099 let _ = fs::remove_file(&p);
2100 }
2101
2102 #[test]
2105 fn test_sampling_temperature_zero_is_argmax() {
2106 let logits = vec![0.1f32, 0.5, 0.9, 0.3, 0.2];
2109 let idx = execute_sampling(&logits, 0.0, 5);
2110 assert_eq!(
2111 idx, 2,
2112 "argmax of [0.1,0.5,0.9,0.3,0.2] should be 2, got {}",
2113 idx
2114 );
2115
2116 let logits2 = vec![10.0f32, 1.0, 2.0];
2118 let idx2 = execute_sampling(&logits2, 0.0, 3);
2119 assert_eq!(idx2, 0, "argmax of [10,1,2] should be 0, got {}", idx2);
2120 }
2121
2122 #[test]
2123 fn test_sampling_topk_one_is_argmax() {
2124 let logits = vec![0.1f32, 0.8, 0.3, 0.6];
2127 let idx = execute_sampling(&logits, 1.0, 1);
2128 assert_eq!(idx, 1, "top_k=1 should return argmax (1), got {}", idx);
2129
2130 let logits2 = vec![3.0f32, 1.0, 2.0, 5.0, 4.0];
2131 let idx2 = execute_sampling(&logits2, 2.0, 1);
2132 assert_eq!(idx2, 3, "top_k=1 should return argmax (3), got {}", idx2);
2133 }
2134}