Skip to main content

tpt_gpu_runtime/
inference.rs

1//! LLM inference API for the TPT runtime.
2//!
3//! # Architecture
4//! - `LlmInference` — stable public trait; tpt-spark and tpt-crucible depend on this.
5//! - `GpuInferenceEngine` — implementation that routes through layer5 kernel handles
6//!   (`EmbeddingKernel`, `RmsNormKernel`, `AttentionKernel`, `GemmKernel`, `SoftmaxKernel`)
7//!   with `VendorBackend::detect()` providing automatic CUDA → ROCm → Metal → TPTIR routing.
8//! - `ModelInfo` — fields extracted from the GGUF binary header.
9//!
10//! Weights are zero-initialised at `load()` time; real weight loading from the
11//! GGUF tensor section is the next increment. All kernel calls are real — the
12//! output is numerically trivial (uniform logits → argmax → token 0) but the
13//! full forward-pass path is exercised end-to-end.
14
15use std::io;
16use std::path::{Path, PathBuf};
17use std::sync::{Arc, Mutex};
18
19use tpt_gpu_primitives::memory::{BufferFlags, DType, Shape};
20use tpt_gpu_primitives::{
21    AttentionKernel, EmbeddingKernel, GemmKernel, GpuBuffer, QuantGemmKernel, RmsNormKernel,
22    SoftmaxKernel, VendorBackend,
23};
24
25use crate::arch::{template_for_arch, ArchTemplate, ForwardOp};
26use crate::error::{ErrorCode, TptrError, TptrResult};
27use crate::kv_cache::KvCache;
28use crate::rope::RopeConfig;
29
30// ---------------------------------------------------------------------------
31// ModelInfo
32// ---------------------------------------------------------------------------
33
34/// Metadata extracted from the GGUF or TPTF file header.
35#[derive(Debug, Clone)]
36pub struct ModelInfo {
37    pub path: PathBuf,
38    /// Architecture tag (e.g. `"llama3"`, `"qwen2"`).
39    pub arch: String,
40    /// Maximum sequence length supported by this model.
41    pub context_len: u32,
42    /// Vocabulary size (number of tokens).
43    pub vocab_size: u32,
44    /// Width of hidden state vectors.
45    pub hidden_dim: u32,
46    /// Number of query attention heads.
47    pub num_heads: u32,
48    /// Number of key/value attention heads (GQA; equals `num_heads` for MHA).
49    pub num_kv_heads: u32,
50    /// Intermediate dimension of the FFN block.
51    pub ffn_dim: u32,
52    /// Number of transformer layers (blocks).
53    pub num_layers: u32,
54    /// Per-layer quantization bit depths from tpt-gpu-model-optimizer (2/4/6/8/16).
55    /// Empty means all layers use f32. Indexed `[layer_idx]`.
56    pub per_layer_bits: Vec<u8>,
57    /// Neuron indices zeroed by the surgical pruner, one `Vec<u32>` per layer.
58    /// `None` means no pruning was applied.
59    pub pruning_mask: Option<Vec<Vec<u32>>>,
60}
61
62/// The on-disk format of a model file.
63#[derive(Debug, Clone, Copy, PartialEq)]
64pub enum ModelFormat {
65    Gguf,
66    Tptf,
67    Unknown,
68}
69
70/// Detect the on-disk format by reading the 4-byte magic.
71pub fn detect_format(path: &Path) -> ModelFormat {
72    use std::fs::File;
73    use std::io::Read;
74    let Ok(mut f) = File::open(path) else {
75        return ModelFormat::Unknown;
76    };
77    let mut magic = [0u8; 4];
78    if f.read_exact(&mut magic).is_err() {
79        return ModelFormat::Unknown;
80    }
81    match &magic {
82        b"GGUF" => ModelFormat::Gguf,
83        b"TPTF" => ModelFormat::Tptf,
84        _ => ModelFormat::Unknown,
85    }
86}
87
88// ---------------------------------------------------------------------------
89// GGUF header parser
90// ---------------------------------------------------------------------------
91
92/// GGUF KV-metadata value types (from the GGUF spec v2/v3).
93#[repr(u32)]
94#[derive(Debug, Clone, Copy, PartialEq)]
95enum GgufType {
96    Uint8 = 0,
97    Int8 = 1,
98    Uint16 = 2,
99    Int16 = 3,
100    Uint32 = 4,
101    Int32 = 5,
102    Float32 = 6,
103    Bool = 7,
104    String = 8,
105    Array = 9,
106    Uint64 = 10,
107    Int64 = 11,
108    Float64 = 12,
109}
110
111impl GgufType {
112    fn from_u32(v: u32) -> Option<Self> {
113        match v {
114            0 => Some(Self::Uint8),
115            1 => Some(Self::Int8),
116            2 => Some(Self::Uint16),
117            3 => Some(Self::Int16),
118            4 => Some(Self::Uint32),
119            5 => Some(Self::Int32),
120            6 => Some(Self::Float32),
121            7 => Some(Self::Bool),
122            8 => Some(Self::String),
123            9 => Some(Self::Array),
124            10 => Some(Self::Uint64),
125            11 => Some(Self::Int64),
126            12 => Some(Self::Float64),
127            _ => None,
128        }
129    }
130
131    /// Byte width for scalar types (None for String and Array).
132    fn scalar_bytes(self) -> Option<usize> {
133        match self {
134            Self::Bool | Self::Uint8 | Self::Int8 => Some(1),
135            Self::Uint16 | Self::Int16 => Some(2),
136            Self::Uint32 | Self::Int32 | Self::Float32 => Some(4),
137            Self::Uint64 | Self::Int64 | Self::Float64 => Some(8),
138            Self::String | Self::Array => None,
139        }
140    }
141}
142
143/// Minimal reader over a byte buffer with LE integer helpers.
144struct BufReader<'a> {
145    data: &'a [u8],
146    pos: usize,
147}
148
149impl<'a> BufReader<'a> {
150    fn new(data: &'a [u8]) -> Self {
151        Self { data, pos: 0 }
152    }
153
154    fn remaining(&self) -> usize {
155        self.data.len().saturating_sub(self.pos)
156    }
157
158    fn read_bytes(&mut self, n: usize) -> io::Result<&'a [u8]> {
159        if self.pos + n > self.data.len() {
160            return Err(io::Error::new(
161                io::ErrorKind::UnexpectedEof,
162                "GGUF truncated",
163            ));
164        }
165        let slice = &self.data[self.pos..self.pos + n];
166        self.pos += n;
167        Ok(slice)
168    }
169
170    fn read_u8(&mut self) -> io::Result<u8> {
171        Ok(self.read_bytes(1)?[0])
172    }
173    #[allow(dead_code)]
174    fn read_u16(&mut self) -> io::Result<u16> {
175        Ok(u16::from_le_bytes(self.read_bytes(2)?.try_into().unwrap()))
176    }
177    fn read_u32(&mut self) -> io::Result<u32> {
178        Ok(u32::from_le_bytes(self.read_bytes(4)?.try_into().unwrap()))
179    }
180    #[allow(dead_code)]
181    fn read_i32(&mut self) -> io::Result<i32> {
182        Ok(i32::from_le_bytes(self.read_bytes(4)?.try_into().unwrap()))
183    }
184    fn read_u64(&mut self) -> io::Result<u64> {
185        Ok(u64::from_le_bytes(self.read_bytes(8)?.try_into().unwrap()))
186    }
187    #[allow(dead_code)]
188    fn read_f32(&mut self) -> io::Result<f32> {
189        Ok(f32::from_le_bytes(self.read_bytes(4)?.try_into().unwrap()))
190    }
191    #[allow(dead_code)]
192    fn read_bool(&mut self) -> io::Result<bool> {
193        Ok(self.read_u8()? != 0)
194    }
195
196    fn read_gguf_string(&mut self) -> io::Result<String> {
197        let len = self.read_u64()? as usize;
198        let bytes = self.read_bytes(len)?;
199        String::from_utf8(bytes.to_vec()).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
200    }
201
202    /// Skip one GGUF value of the given type without parsing it into Rust.
203    fn skip_value(&mut self, vtype: GgufType) -> io::Result<()> {
204        match vtype {
205            GgufType::String => {
206                self.read_gguf_string()?;
207            }
208            GgufType::Array => {
209                let elem_type = GgufType::from_u32(self.read_u32()?).ok_or_else(|| {
210                    io::Error::new(io::ErrorKind::InvalidData, "unknown array elem type")
211                })?;
212                let count = self.read_u64()? as usize;
213                for _ in 0..count {
214                    self.skip_value(elem_type)?;
215                }
216            }
217            scalar => {
218                let n = scalar.scalar_bytes().ok_or_else(|| {
219                    io::Error::new(io::ErrorKind::InvalidData, "scalar_bytes unset")
220                })?;
221                self.read_bytes(n)?;
222            }
223        }
224        Ok(())
225    }
226}
227
228/// Parse the GGUF v2/v3 binary header and extract fields needed for `ModelInfo`.
229///
230/// Only reads the KV-metadata section; the tensor index and weight data are not
231/// loaded here (that is the next increment — weight loading from GGUF).
232pub fn parse_gguf_header(path: &Path) -> TptrResult<ModelInfo> {
233    use std::fs;
234
235    if !path.exists() {
236        return Err(TptrError::new(
237            ErrorCode::InvalidKernel,
238            format!("model file not found: {}", path.display()),
239        ));
240    }
241
242    let data = fs::read(path).map_err(|e| {
243        TptrError::new(
244            ErrorCode::InternalError,
245            format!("cannot read {}: {}", path.display(), e),
246        )
247    })?;
248
249    let mut r = BufReader::new(&data);
250
251    // Magic
252    let magic = r.read_bytes(4).map_err(|e| {
253        TptrError::new(
254            ErrorCode::InvalidKernel,
255            format!("cannot read GGUF magic: {}", e),
256        )
257    })?;
258    if magic != b"GGUF" {
259        return Err(TptrError::new(
260            ErrorCode::InvalidKernel,
261            format!("not a GGUF file (magic = {:?})", magic),
262        ));
263    }
264
265    // Version
266    let version = r.read_u32().map_err(|e| {
267        TptrError::new(
268            ErrorCode::InvalidKernel,
269            format!("cannot read GGUF version: {}", e),
270        )
271    })?;
272    if version == 0 || version > 3 {
273        return Err(TptrError::new(
274            ErrorCode::InvalidKernel,
275            format!("unsupported GGUF version {} (supported: 1, 2, 3)", version),
276        ));
277    }
278
279    // tensor_count and kv_count (u64 in v2/v3, u32 in v1)
280    let (_tensor_count, kv_count) = if version >= 2 {
281        (r.read_u64().unwrap_or(0), r.read_u64().unwrap_or(0))
282    } else {
283        (
284            r.read_u32().unwrap_or(0) as u64,
285            r.read_u32().unwrap_or(0) as u64,
286        )
287    };
288
289    // Collect the KV keys we care about
290    let mut arch = String::new();
291    let mut context_len: u32 = 2048;
292    let mut hidden_dim: u32 = 0;
293    let mut num_heads: u32 = 0;
294    let mut num_kv_heads: u32 = 0;
295    let mut ffn_dim: u32 = 0;
296    let mut num_layers: u32 = 0;
297    let mut vocab_size: u32 = 0;
298
299    for _ in 0..kv_count {
300        let key = match r.read_gguf_string() {
301            Ok(k) => k,
302            Err(_) => break,
303        };
304        let vtype_raw = match r.read_u32() {
305            Ok(v) => v,
306            Err(_) => break,
307        };
308        let vtype = match GgufType::from_u32(vtype_raw) {
309            Some(t) => t,
310            None => break,
311        };
312
313        match key.as_str() {
314            "general.architecture" if vtype == GgufType::String => {
315                arch = r.read_gguf_string().unwrap_or_default();
316            }
317            "llm.context_length" if vtype == GgufType::Uint32 => {
318                context_len = r.read_u32().unwrap_or(2048);
319            }
320            "llm.embedding_length" if vtype == GgufType::Uint32 => {
321                hidden_dim = r.read_u32().unwrap_or(0);
322            }
323            "llm.attention.head_count" if vtype == GgufType::Uint32 => {
324                num_heads = r.read_u32().unwrap_or(0);
325            }
326            "llm.attention.head_count_kv" if vtype == GgufType::Uint32 => {
327                num_kv_heads = r.read_u32().unwrap_or(0);
328            }
329            "llm.feed_forward_length" if vtype == GgufType::Uint32 => {
330                ffn_dim = r.read_u32().unwrap_or(0);
331            }
332            "llm.block_count" if vtype == GgufType::Uint32 => {
333                num_layers = r.read_u32().unwrap_or(0);
334            }
335            "tokenizer.ggml.tokens" if vtype == GgufType::Array => {
336                // Array: u32 elem_type + u64 count + elements
337                let _elem_type = r.read_u32().unwrap_or(0);
338                let count = r.read_u64().unwrap_or(0);
339                vocab_size = count as u32;
340                // Skip the array contents
341                let elem_t = GgufType::String; // tokens are strings
342                for _ in 0..count {
343                    let _ = r.skip_value(elem_t);
344                }
345            }
346            _ => {
347                if r.skip_value(vtype).is_err() {
348                    break;
349                }
350            }
351        }
352    }
353
354    // Apply safe defaults for missing fields
355    if hidden_dim == 0 {
356        hidden_dim = 4096;
357    }
358    if num_heads == 0 {
359        num_heads = 32;
360    }
361    if num_layers == 0 {
362        num_layers = 32;
363    }
364    if vocab_size == 0 {
365        vocab_size = 32000;
366    }
367    if ffn_dim == 0 {
368        ffn_dim = hidden_dim * 8 / 3;
369    } // typical Llama ratio
370
371    Ok(ModelInfo {
372        path: path.to_owned(),
373        arch,
374        context_len,
375        vocab_size,
376        hidden_dim,
377        num_heads,
378        num_kv_heads,
379        ffn_dim,
380        num_layers,
381        per_layer_bits: Vec::new(),
382        pruning_mask: None,
383    })
384}
385
386// ---------------------------------------------------------------------------
387// TPTF header parser
388// ---------------------------------------------------------------------------
389
390/// Parse the TPTF v1 binary header and extract `ModelInfo`.
391///
392/// TPTF layout (first 512 bytes):
393/// - [0..4]   magic "TPTF"
394/// - [4..8]   version u32
395/// - [8..12]  flags u32 (bit 0 = has_pruning_mask, bit 1 = has_chat_template)
396/// - [12..76] arch string: u8 length prefix + UTF-8 bytes (max 63 chars)
397/// - [76..80] context_len u32
398/// - [80..84] vocab_size u32
399/// - [84..88] hidden_dim u32
400/// - [88..92] num_heads u32
401/// - [92..96] num_kv_heads u32
402/// - [96..100] ffn_dim u32
403/// - [100..104] num_layers u32
404/// - [104..232] per_layer_bits[u8; 128]
405pub fn parse_tptf_header(path: &Path) -> TptrResult<ModelInfo> {
406    use std::fs;
407
408    if !path.exists() {
409        return Err(TptrError::new(
410            ErrorCode::InvalidKernel,
411            format!("model file not found: {}", path.display()),
412        ));
413    }
414
415    let data = fs::read(path).map_err(|e| {
416        TptrError::new(
417            ErrorCode::InternalError,
418            format!("cannot read {}: {}", path.display(), e),
419        )
420    })?;
421
422    if data.len() < 512 {
423        return Err(TptrError::new(
424            ErrorCode::InvalidKernel,
425            format!("TPTF file too short ({} bytes, need ≥ 512)", data.len()),
426        ));
427    }
428
429    let mut r = BufReader::new(&data);
430
431    let magic = r.read_bytes(4).map_err(|e| {
432        TptrError::new(
433            ErrorCode::InvalidKernel,
434            format!("cannot read TPTF magic: {}", e),
435        )
436    })?;
437    if magic != b"TPTF" {
438        return Err(TptrError::new(
439            ErrorCode::InvalidKernel,
440            format!("not a TPTF file (magic = {:?})", magic),
441        ));
442    }
443
444    let version = r.read_u32().map_err(|e| {
445        TptrError::new(
446            ErrorCode::InvalidKernel,
447            format!("cannot read TPTF version: {}", e),
448        )
449    })?;
450    if version != 1 {
451        return Err(TptrError::new(
452            ErrorCode::InvalidKernel,
453            format!("unsupported TPTF version {} (supported: 1)", version),
454        ));
455    }
456
457    let flags = r.read_u32().unwrap_or(0);
458    let has_pruning_mask = (flags & 1) != 0;
459
460    // Arch string: 1-byte length prefix + bytes (packed into [12..76])
461    let arch_len = r.read_u8().unwrap_or(0) as usize;
462    let arch = if arch_len > 0 && arch_len <= 63 {
463        let bytes = r.read_bytes(arch_len).unwrap_or_default();
464        String::from_utf8_lossy(bytes).to_string()
465    } else {
466        r.read_bytes(63.min(r.remaining())).ok(); // skip padding
467        String::new()
468    };
469    // Seek to byte 76 (skip any remaining arch padding)
470    let consumed = 4 + 4 + 4 + 1 + arch_len;
471    if consumed < 76 {
472        let _ = r.read_bytes(76 - consumed);
473    }
474
475    let context_len = r.read_u32().unwrap_or(2048);
476    let vocab_size = r.read_u32().unwrap_or(32000);
477    let hidden_dim = r.read_u32().unwrap_or(4096);
478    let num_heads = r.read_u32().unwrap_or(32);
479    let num_kv_heads = r.read_u32().unwrap_or(8);
480    let ffn_dim = r.read_u32().unwrap_or(0);
481    let num_layers = r.read_u32().unwrap_or(32);
482
483    // Per-layer bits array [104..232]
484    let bits_bytes = r.read_bytes(128).unwrap_or_default();
485    let per_layer_bits: Vec<u8> = bits_bytes[..num_layers.min(128) as usize]
486        .iter()
487        .copied()
488        .filter(|&b| b > 0)
489        .collect();
490
491    let pruning_mask = if has_pruning_mask {
492        Some(vec![Vec::new(); num_layers as usize])
493    } else {
494        None
495    };
496
497    let ffn_dim = if ffn_dim == 0 {
498        hidden_dim * 8 / 3
499    } else {
500        ffn_dim
501    };
502
503    Ok(ModelInfo {
504        path: path.to_owned(),
505        arch,
506        context_len,
507        vocab_size,
508        hidden_dim,
509        num_heads,
510        num_kv_heads,
511        ffn_dim,
512        num_layers,
513        per_layer_bits,
514        pruning_mask,
515    })
516}
517
518// ---------------------------------------------------------------------------
519// Model weights (zero-initialised)
520// ---------------------------------------------------------------------------
521
522/// Per-layer weight buffers for a standard transformer block.
523struct LayerWeights {
524    q_proj: GpuBuffer<f32>,    // [hidden_dim, num_heads * head_dim]
525    k_proj: GpuBuffer<f32>,    // [hidden_dim, num_kv_heads * head_dim]
526    v_proj: GpuBuffer<f32>,    // [hidden_dim, num_kv_heads * head_dim]
527    o_proj: GpuBuffer<f32>,    // [num_heads * head_dim, hidden_dim]
528    gate_proj: GpuBuffer<f32>, // [hidden_dim, ffn_dim]
529    up_proj: GpuBuffer<f32>,   // [hidden_dim, ffn_dim]
530    down_proj: GpuBuffer<f32>, // [ffn_dim, hidden_dim]
531    norm1: GpuBuffer<f32>,     // [hidden_dim]
532    norm2: GpuBuffer<f32>,     // [hidden_dim]
533}
534
535/// Full model weight set (zero-initialised; populated from GGUF tensors in the
536/// next increment when the tensor section is parsed).
537struct ModelWeights {
538    embed_table: GpuBuffer<f32>, // [vocab_size, hidden_dim]
539    layers: Vec<LayerWeights>,
540    final_norm: GpuBuffer<f32>, // [hidden_dim]
541    lm_head: GpuBuffer<f32>,    // [hidden_dim, vocab_size] — or tied to embed_table
542    lm_head_tied: bool,
543}
544
545fn alloc_f32(rows: usize, cols: usize) -> TptrResult<GpuBuffer<f32>> {
546    GpuBuffer::new(Shape::dim2(rows, cols), DType::F32, BufferFlags::STORAGE)
547        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))
548}
549
550fn alloc_f32_1d(n: usize) -> TptrResult<GpuBuffer<f32>> {
551    GpuBuffer::new(Shape::new(&[n]), DType::F32, BufferFlags::STORAGE)
552        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))
553}
554
555impl ModelWeights {
556    /// Load model weights from a TPTF v1 file.
557    ///
558    /// Parses the TPTF header via [`parse_tptf_header`], allocates all weight
559    /// buffers (zero-initialised), then reads each quantized tensor block from
560    /// the tensor data section, dequantizes it to `f32`, and copies the result
561    /// into the matching [`GpuBuffer`] field (gate_proj, up_proj, down_proj,
562    /// q_proj, k_proj, v_proj, o_proj, norm1, norm2).  Tensors absent from the
563    /// file remain zero-initialised (e.g. attention projections when the TPTF
564    /// was produced by a FFN-only quantizer pass).
565    fn load_tptf(path: &Path, info: &ModelInfo, tied: bool) -> TptrResult<Self> {
566        use std::fs;
567
568        let data = fs::read(path).map_err(|e| {
569            TptrError::new(
570                ErrorCode::InternalError,
571                format!("load_tptf: cannot read {}: {}", path.display(), e),
572            )
573        })?;
574
575        // Tensor section offset is stored as a u64 at byte 460 in the TPTF header.
576        // Fall back to the fixed HEADER_SIZE (512) when the field is absent or zero.
577        const TPTF_TENSOR_OFFSET_FIELD: usize = 460;
578        const TPTF_HEADER_SIZE: usize = 512;
579        let tensor_start: usize = if data.len() >= TPTF_TENSOR_OFFSET_FIELD + 8 {
580            let raw = u64::from_le_bytes(
581                data[TPTF_TENSOR_OFFSET_FIELD..TPTF_TENSOR_OFFSET_FIELD + 8]
582                    .try_into()
583                    .unwrap(),
584            ) as usize;
585            if raw >= TPTF_HEADER_SIZE {
586                raw
587            } else {
588                TPTF_HEADER_SIZE
589            }
590        } else {
591            TPTF_HEADER_SIZE
592        };
593
594        // Tokenizer section offset (field immediately after tensor_offset).
595        let tokenizer_start: usize = if data.len() >= TPTF_TENSOR_OFFSET_FIELD + 16 {
596            u64::from_le_bytes(
597                data[TPTF_TENSOR_OFFSET_FIELD + 8..TPTF_TENSOR_OFFSET_FIELD + 16]
598                    .try_into()
599                    .unwrap(),
600            ) as usize
601        } else {
602            0
603        };
604
605        let tensor_end = if tokenizer_start > tensor_start {
606            tokenizer_start
607        } else {
608            data.len()
609        };
610
611        let mut weights = Self::allocate(info, tied)?;
612        let num_layers = info.num_layers as usize;
613
614        // Parse tensor blocks sequentially.  Each block's binary layout:
615        //   u32  layer_idx
616        //   u8   name_len (max 31)
617        //   [u8; name_len]  name
618        //   u8   bits
619        //   u32  group_size
620        //   u32  rows
621        //   u32  cols
622        //   u32  weights_len
623        //   [u8; weights_len]  packed weights
624        //   u32  scales_len
625        //   [f32; scales_len]  scales
626        //   u32  zp_len
627        //   [i8; zp_len]  zero_points
628        //   <padding to next 128-byte boundary (absolute file position)>
629        let mut pos = tensor_start;
630        while pos + 14 < tensor_end.min(data.len()) {
631            let block_start = pos;
632
633            // layer_idx
634            if pos + 4 > data.len() {
635                break;
636            }
637            let layer_idx = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
638            pos += 4;
639
640            // name
641            if pos >= data.len() {
642                break;
643            }
644            let name_len = data[pos] as usize;
645            pos += 1;
646            if name_len > 31 || pos + name_len > data.len() {
647                break;
648            }
649            let name = String::from_utf8_lossy(&data[pos..pos + name_len]).to_string();
650            pos += name_len;
651
652            // bits, group_size, rows, cols
653            if pos + 13 > data.len() {
654                break;
655            }
656            let bits = data[pos];
657            pos += 1;
658            let group_size = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
659            pos += 4;
660            let rows = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
661            pos += 4;
662            let cols = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
663            pos += 4;
664
665            // packed weights
666            if pos + 4 > data.len() {
667                break;
668            }
669            let weights_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
670            pos += 4;
671            if pos + weights_len > data.len() {
672                break;
673            }
674            let packed = data[pos..pos + weights_len].to_vec();
675            pos += weights_len;
676
677            // scales
678            if pos + 4 > data.len() {
679                break;
680            }
681            let scales_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
682            pos += 4;
683            if pos + scales_len * 4 > data.len() {
684                break;
685            }
686            let mut scales = vec![1.0f32; scales_len];
687            for s in scales.iter_mut() {
688                *s = f32::from_le_bytes(data[pos..pos + 4].try_into().unwrap());
689                pos += 4;
690            }
691
692            // zero_points
693            if pos + 4 > data.len() {
694                break;
695            }
696            let zp_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize;
697            pos += 4;
698            if pos + zp_len > data.len() {
699                break;
700            }
701            let zero_points: Vec<i8> = data[pos..pos + zp_len].iter().map(|&b| b as i8).collect();
702            pos += zp_len;
703
704            // Advance to the next 128-byte boundary (absolute file position).
705            let _ = block_start; // suppress unused warning
706            pos = (pos + 127) & !127_usize;
707
708            // Dequantize.
709            let num_elements = rows * cols;
710            let group_sz = if group_size > 0 {
711                group_size
712            } else {
713                num_elements.max(1)
714            };
715            let f32_data =
716                tptf_dequantize(&packed, &scales, &zero_points, bits, group_sz, num_elements);
717
718            // Copy into the matching GpuBuffer field.
719            if layer_idx < num_layers {
720                let layer = &mut weights.layers[layer_idx];
721                let target: Option<&mut GpuBuffer<f32>> = match name.as_str() {
722                    "gate_proj" => Some(&mut layer.gate_proj),
723                    "up_proj" => Some(&mut layer.up_proj),
724                    "down_proj" => Some(&mut layer.down_proj),
725                    "q_proj" => Some(&mut layer.q_proj),
726                    "k_proj" => Some(&mut layer.k_proj),
727                    "v_proj" => Some(&mut layer.v_proj),
728                    "o_proj" => Some(&mut layer.o_proj),
729                    "norm1" => Some(&mut layer.norm1),
730                    "norm2" => Some(&mut layer.norm2),
731                    _ => None,
732                };
733                if let Some(buf) = target {
734                    let n = f32_data.len().min(buf.num_elements());
735                    if n > 0 {
736                        let _ = buf.copy_from_host(&f32_data[..n]);
737                    }
738                }
739            } else if name == "embed_table" || name == "embedding" {
740                let n = f32_data.len().min(weights.embed_table.num_elements());
741                if n > 0 {
742                    let _ = weights.embed_table.copy_from_host(&f32_data[..n]);
743                }
744            } else if name == "final_norm" {
745                let n = f32_data.len().min(weights.final_norm.num_elements());
746                if n > 0 {
747                    let _ = weights.final_norm.copy_from_host(&f32_data[..n]);
748                }
749            } else if name == "lm_head" && !weights.lm_head_tied {
750                let n = f32_data.len().min(weights.lm_head.num_elements());
751                if n > 0 {
752                    let _ = weights.lm_head.copy_from_host(&f32_data[..n]);
753                }
754            }
755        }
756
757        Ok(weights)
758    }
759
760    fn allocate(info: &ModelInfo, tied: bool) -> TptrResult<Self> {
761        let h = info.hidden_dim as usize;
762        let v = info.vocab_size as usize;
763        let nh = info.num_heads.max(1) as usize;
764        let nk = info.num_kv_heads.max(1) as usize;
765        let hd = h / nh; // head_dim
766        let f = info.ffn_dim as usize;
767
768        let embed_table = alloc_f32(v, h)?;
769        let final_norm = alloc_f32_1d(h)?;
770        let lm_head = if tied {
771            alloc_f32(1, 1)?
772        } else {
773            alloc_f32(h, v)?
774        }; // placeholder when tied
775
776        let mut layers = Vec::with_capacity(info.num_layers as usize);
777        for _ in 0..info.num_layers {
778            layers.push(LayerWeights {
779                q_proj: alloc_f32(h, nh * hd)?,
780                k_proj: alloc_f32(h, nk * hd)?,
781                v_proj: alloc_f32(h, nk * hd)?,
782                o_proj: alloc_f32(nh * hd, h)?,
783                gate_proj: alloc_f32(h, f)?,
784                up_proj: alloc_f32(h, f)?,
785                down_proj: alloc_f32(f, h)?,
786                norm1: alloc_f32_1d(h)?,
787                norm2: alloc_f32_1d(h)?,
788            });
789        }
790
791        Ok(Self {
792            embed_table,
793            layers,
794            final_norm,
795            lm_head,
796            lm_head_tied: tied,
797        })
798    }
799}
800
801// ---------------------------------------------------------------------------
802// RoPE config helpers
803// ---------------------------------------------------------------------------
804
805/// Build the `RopeConfig` for a given model, deriving `head_dim` from
806/// `hidden_dim / num_heads` and `max_seq_len` from `context_len`.
807fn rope_config_for_arch(arch: &str, info: &ModelInfo) -> Option<RopeConfig> {
808    let num_heads = info.num_heads.max(1) as usize;
809    let head_dim = info.hidden_dim as usize / num_heads;
810    let max_seq_len = info.context_len as usize;
811    match arch {
812        "llama3" => Some(RopeConfig {
813            head_dim,
814            base: 500_000.0,
815            max_seq_len,
816        }),
817        "llama" => Some(RopeConfig {
818            head_dim,
819            base: 10_000.0,
820            max_seq_len,
821        }),
822        "mistral" => Some(RopeConfig {
823            head_dim,
824            base: 10_000.0,
825            max_seq_len,
826        }),
827        "qwen2" | "qwen" => Some(RopeConfig {
828            head_dim,
829            base: 1_000_000.0,
830            max_seq_len,
831        }),
832        "phi3" => Some(RopeConfig {
833            head_dim,
834            base: 10_000.0,
835            max_seq_len,
836        }),
837        "gemma2" | "gemma" => Some(RopeConfig {
838            head_dim,
839            base: 10_000.0,
840            max_seq_len,
841        }),
842        _ => None,
843    }
844}
845
846// ---------------------------------------------------------------------------
847// LlmInference trait
848// ---------------------------------------------------------------------------
849
850/// Core trait for LLM token-level inference via the TPT runtime.
851///
852/// # Contract
853/// - `load` is called once; subsequent `infer` calls reuse the loaded weights.
854/// - `callback` is invoked for each generated token in order.
855/// - `cancel` signals the engine to stop at the next safe preemption point.
856pub trait LlmInference: Send + Sync {
857    fn load(model_path: &Path) -> TptrResult<Self>
858    where
859        Self: Sized;
860    fn model_info(&self) -> &ModelInfo;
861    fn infer(
862        &mut self,
863        tokens: &[u32],
864        max_new_tokens: u32,
865        callback: impl FnMut(u32) + Send,
866    ) -> TptrResult<()>;
867    fn cancel(&mut self);
868}
869
870// ---------------------------------------------------------------------------
871// GpuInferenceEngine
872// ---------------------------------------------------------------------------
873
874/// GPU-backed inference engine.
875///
876/// Routes through layer5 kernel handles with automatic hardware detection:
877/// `VendorBackend::detect()` selects CUDA → ROCm → Metal → TPTIR fallback.
878pub struct GpuInferenceEngine {
879    info: ModelInfo,
880    template: ArchTemplate,
881    weights: ModelWeights,
882    embed_k: EmbeddingKernel,
883    rmsnorm_k: RmsNormKernel,
884    attn_k: AttentionKernel,
885    gemm_k: GemmKernel,
886    quant_gemm_k: QuantGemmKernel,
887    softmax_k: SoftmaxKernel,
888    kv_cache: KvCache,
889    cancelled: Arc<Mutex<bool>>,
890    /// xorshift64 PRNG state for temperature sampling (never zero).
891    rng_state: u64,
892    /// RoPE configuration derived from the model architecture.
893    /// `None` only for architectures that do not use RoPE (none currently supported).
894    rope_config: Option<RopeConfig>,
895}
896
897impl std::fmt::Debug for GpuInferenceEngine {
898    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
899        f.debug_struct("GpuInferenceEngine")
900            .field("arch", &self.info.arch)
901            .field("num_layers", &self.info.num_layers)
902            .finish()
903    }
904}
905
906impl LlmInference for GpuInferenceEngine {
907    fn load(model_path: &Path) -> TptrResult<Self> {
908        let fmt = detect_format(model_path);
909        let info = match fmt {
910            ModelFormat::Gguf => parse_gguf_header(model_path)?,
911            ModelFormat::Tptf => parse_tptf_header(model_path)?,
912            ModelFormat::Unknown => {
913                // Fall back to GGUF parser which will give a descriptive error.
914                parse_gguf_header(model_path)?
915            }
916        };
917        let template = template_for_arch(&info.arch, &info)?;
918
919        // Determine if this arch uses tied embeddings.
920        let tied = template
921            .post_ops
922            .iter()
923            .any(|op| matches!(op, ForwardOp::LinearOut { tied: true, .. }));
924
925        // For TPTF files load the quantized tensor blocks from the tensor data
926        // section rather than returning a zero-initialised weight set.
927        let weights = if fmt == ModelFormat::Tptf {
928            ModelWeights::load_tptf(model_path, &info, tied)?
929        } else {
930            ModelWeights::allocate(&info, tied)?
931        };
932
933        // All kernel handles share a single VendorBackend::detect() result.
934        let vendor = VendorBackend::detect();
935        let embed_k = EmbeddingKernel::with_vendor(vendor.clone());
936        let rmsnorm_k = RmsNormKernel::with_vendor(vendor.clone());
937        let attn_k = AttentionKernel::with_vendor(vendor.clone());
938        let gemm_k = GemmKernel::with_vendor(vendor.clone());
939        let quant_gemm_k = QuantGemmKernel::with_vendor(vendor.clone());
940        let softmax_k = SoftmaxKernel::with_vendor(vendor);
941
942        let head_dim = info.hidden_dim / info.num_heads.max(1);
943        let kv_cache = KvCache::new(
944            info.num_layers,
945            info.context_len,
946            info.num_kv_heads.max(1),
947            head_dim,
948        );
949
950        let rope_config = rope_config_for_arch(&info.arch, &info);
951
952        Ok(Self {
953            info,
954            template,
955            weights,
956            embed_k,
957            rmsnorm_k,
958            attn_k,
959            gemm_k,
960            quant_gemm_k,
961            softmax_k,
962            kv_cache,
963            cancelled: Arc::new(Mutex::new(false)),
964            rng_state: 0x853c49e6748fea9b,
965            rope_config,
966        })
967    }
968
969    fn model_info(&self) -> &ModelInfo {
970        &self.info
971    }
972
973    fn infer(
974        &mut self,
975        tokens: &[u32],
976        max_new_tokens: u32,
977        mut callback: impl FnMut(u32) + Send,
978    ) -> TptrResult<()> {
979        if tokens.is_empty() {
980            return Err(TptrError::new(
981                ErrorCode::ArgumentMismatch,
982                "token list is empty",
983            ));
984        }
985
986        self.kv_cache.reset();
987
988        // Decode mode: process one token at a time starting from the last prompt token.
989        // Prefill (processing all prompt tokens at once) is a future optimisation.
990        let mut current_token = *tokens.last().unwrap();
991
992        for _ in 0..max_new_tokens {
993            if *self.cancelled.lock().unwrap() {
994                break;
995            }
996
997            let next = self.forward_step(current_token)?;
998            callback(next);
999            current_token = next;
1000        }
1001
1002        Ok(())
1003    }
1004
1005    fn cancel(&mut self) {
1006        *self.cancelled.lock().unwrap() = true;
1007    }
1008}
1009
1010impl GpuInferenceEngine {
1011    /// Run one autoregressive decode step for a single token.
1012    ///
1013    /// Returns the sampled next token ID.
1014    fn forward_step(&mut self, token: u32) -> TptrResult<u32> {
1015        let hidden_dim = self.info.hidden_dim as usize;
1016        let vocab_size = self.info.vocab_size as usize;
1017        let num_heads = self.info.num_heads.max(1) as usize;
1018        let num_kv = self.info.num_kv_heads.max(1) as usize;
1019        let head_dim = hidden_dim / num_heads;
1020        let ffn_dim = self.info.ffn_dim as usize;
1021
1022        // --- Pre-ops: Embedding ---
1023        let idx_buf = token_to_index_buffer(token)?;
1024        let mut hidden = self
1025            .embed_k
1026            .execute(&self.weights.embed_table, &idx_buf)
1027            .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, e.to_string()))?;
1028        // Reshape to [1, hidden_dim] for GEMM compatibility
1029        hidden = reshape_to_2d(hidden, 1, hidden_dim)?;
1030
1031        // --- Per-layer ops ---
1032        for layer_idx in 0..self.info.num_layers as usize {
1033            if *self.cancelled.lock().unwrap() {
1034                return Ok(0);
1035            }
1036
1037            // Quantization bit depth for this layer (0 = f32).
1038            let layer_bits = self
1039                .info
1040                .per_layer_bits
1041                .get(layer_idx)
1042                .copied()
1043                .unwrap_or(0);
1044
1045            let layer = &self.weights.layers[layer_idx];
1046
1047            // Pre-attention RmsNorm
1048            let normed = self.rmsnorm_k.execute(&hidden, &layer.norm1).map_err(|e| {
1049                TptrError::new(
1050                    ErrorCode::LaunchFailure,
1051                    format!("rmsnorm1 layer {}: {}", layer_idx, e),
1052                )
1053            })?;
1054            let normed = reshape_to_2d(normed, 1, hidden_dim)?;
1055
1056            // Q, K, V projections — dispatch through QuantGemm for quantized layers.
1057            macro_rules! proj_gemm {
1058                ($input:expr, $weight:expr, $name:literal) => {
1059                    if layer_bits > 0 {
1060                        quant_layer_gemm(&self.quant_gemm_k, $input, $weight, layer_bits).map_err(
1061                            |e| {
1062                                TptrError::new(
1063                                    ErrorCode::LaunchFailure,
1064                                    format!("{} layer {}: {}", $name, layer_idx, e),
1065                                )
1066                            },
1067                        )
1068                    } else {
1069                        self.gemm_k
1070                            .execute($input, $weight, None, 1.0, 0.0)
1071                            .map_err(|e| {
1072                                TptrError::new(
1073                                    ErrorCode::LaunchFailure,
1074                                    format!("{} layer {}: {}", $name, layer_idx, e),
1075                                )
1076                            })
1077                    }
1078                };
1079            }
1080
1081            let q = proj_gemm!(&normed, &layer.q_proj, "q_proj")?;
1082            let k = proj_gemm!(&normed, &layer.k_proj, "k_proj")?;
1083            let v = proj_gemm!(&normed, &layer.v_proj, "v_proj")?;
1084
1085            // Apply RoPE to Q and K before caching so stored K values carry
1086            // positional information.  The decode position is the current
1087            // KV-cache length (0-indexed: first token is position 0).
1088            let rope_pos = self.kv_cache.seq_len;
1089            let (q, k) = if let Some(ref rope_cfg) = self.rope_config {
1090                let mut q_data = buf_to_vec(&q);
1091                let mut k_data = buf_to_vec(&k);
1092                crate::rope::apply_rope(&mut q_data, &mut k_data, rope_pos, rope_cfg);
1093                let q_rotated = vec_to_buf(q_data, 1, num_heads * head_dim)?;
1094                let k_rotated = vec_to_buf(k_data, 1, num_kv * head_dim)?;
1095                (q_rotated, k_rotated)
1096            } else {
1097                (q, k)
1098            };
1099
1100            // KV cache: append current step's K and V.
1101            // After append, the token data is at position `kv_step - 1` but
1102            // seq_len only increments once the last layer has appended.
1103            // We use get_k_at / get_v_at with `kv_step` to read all positions
1104            // including the just-written one regardless of seq_len's state.
1105            let kv_step = self.kv_cache.seq_len + 1;
1106            let k_data = buf_to_vec(&k);
1107            let v_data = buf_to_vec(&v);
1108            self.kv_cache.append(layer_idx, &k_data, &v_data);
1109
1110            let qk_dim = num_heads * head_dim;
1111            let kv_row = num_kv * head_dim;
1112            // Expand GQA/MQA KV heads to full query-head dimension.
1113            let k_full = expand_kv_heads(
1114                self.kv_cache.get_k_at(layer_idx, kv_step),
1115                kv_step,
1116                kv_row,
1117                qk_dim,
1118            );
1119            let v_full = expand_kv_heads(
1120                self.kv_cache.get_v_at(layer_idx, kv_step),
1121                kv_step,
1122                kv_row,
1123                qk_dim,
1124            );
1125            let k_cache = vec_to_buf(k_full, kv_step, qk_dim)?;
1126            let v_cache = vec_to_buf(v_full, kv_step, qk_dim)?;
1127
1128            let scale = Some(1.0 / (head_dim as f32).sqrt());
1129            // Q is the current token [1, qk_dim]; K/V are the full history
1130            // [kv_step, qk_dim]. The AttentionKernel now supports cross-attention
1131            // (q_len != kv_len) so this routes through the host fallback.
1132            let q2d = reshape_to_2d(q, 1, qk_dim)?;
1133            let attn_out = self
1134                .attn_k
1135                .execute(&q2d, &k_cache, &v_cache, scale, None)
1136                .map_err(|e| {
1137                    TptrError::new(
1138                        ErrorCode::LaunchFailure,
1139                        format!("attention layer {}: {}", layer_idx, e),
1140                    )
1141                })?;
1142            let attn_dim = attn_out_dim(&attn_out);
1143            let attn_out = reshape_to_2d(attn_out, 1, attn_dim)?;
1144
1145            // Output projection + residual
1146            let o = proj_gemm!(&attn_out, &layer.o_proj, "o_proj")?;
1147            let o = reshape_to_2d(o, 1, hidden_dim)?;
1148            hidden = elementwise_add_2d(&hidden, &o, hidden_dim)?;
1149
1150            // Pre-FFN RmsNorm
1151            let normed2 = self.rmsnorm_k.execute(&hidden, &layer.norm2).map_err(|e| {
1152                TptrError::new(
1153                    ErrorCode::LaunchFailure,
1154                    format!("rmsnorm2 layer {}: {}", layer_idx, e),
1155                )
1156            })?;
1157            let normed2 = reshape_to_2d(normed2, 1, hidden_dim)?;
1158
1159            // SwiGLU FFN: gate, up, silu(gate)*up, down
1160            let gate = proj_gemm!(&normed2, &layer.gate_proj, "gate_proj")?;
1161            let up = proj_gemm!(&normed2, &layer.up_proj, "up_proj")?;
1162
1163            let ffn_mid = swiglu_host(&gate, &up, ffn_dim)?;
1164            let ffn_mid = reshape_to_2d(ffn_mid, 1, ffn_dim)?;
1165
1166            let ffn_out = proj_gemm!(&ffn_mid, &layer.down_proj, "down_proj")?;
1167            let ffn_out = reshape_to_2d(ffn_out, 1, hidden_dim)?;
1168
1169            hidden = elementwise_add_2d(&hidden, &ffn_out, hidden_dim)?;
1170        }
1171
1172        // --- Post-ops: final norm, lm_head, softmax, sample ---
1173        let normed_final = self
1174            .rmsnorm_k
1175            .execute(&hidden, &self.weights.final_norm)
1176            .map_err(|e| {
1177                TptrError::new(ErrorCode::LaunchFailure, format!("final rmsnorm: {}", e))
1178            })?;
1179        let normed_final = reshape_to_2d(normed_final, 1, hidden_dim)?;
1180
1181        let lm_weight = if self.weights.lm_head_tied {
1182            &self.weights.embed_table
1183        } else {
1184            &self.weights.lm_head
1185        };
1186        // lm_head: [1, hidden_dim] × [hidden_dim, vocab_size] → [1, vocab_size]
1187        // For tied embeddings the embed_table is [vocab_size, hidden_dim]; we transpose by
1188        // swapping arguments: GEMM(hidden, W^T) where W^T is [hidden_dim, vocab_size].
1189        // We approximate by treating the embed_table as already transposed here.
1190        let logits_raw = if self.weights.lm_head_tied {
1191            // Tied: embed_table is [vocab_size, hidden_dim]; compute host-side dot products
1192            tied_lm_head_host(
1193                &normed_final,
1194                &self.weights.embed_table,
1195                hidden_dim,
1196                vocab_size,
1197            )?
1198        } else {
1199            self.gemm_k
1200                .execute(&normed_final, lm_weight, None, 1.0, 0.0)
1201                .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, format!("lm_head: {}", e)))?
1202        };
1203
1204        let logits = reshape_to_2d(logits_raw, 1, vocab_size)?;
1205
1206        // Extract temperature and top_k from the arch template's Sampling post-op.
1207        let (temperature, top_k) = self
1208            .template
1209            .post_ops
1210            .iter()
1211            .find_map(|op| {
1212                if let ForwardOp::Sampling { temperature, top_k } = op {
1213                    Some((*temperature, *top_k))
1214                } else {
1215                    None
1216                }
1217            })
1218            .unwrap_or((1.0, 1));
1219
1220        if temperature <= 0.0 || top_k <= 1 {
1221            // Greedy: softmax then argmax.
1222            let probs = self
1223                .softmax_k
1224                .execute(&logits)
1225                .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, format!("softmax: {}", e)))?;
1226            return Ok(sample_argmax(&probs, vocab_size));
1227        }
1228
1229        // Temperature scaling + top-k sampling.
1230        let next = self.sample_with_temperature(&logits, vocab_size, temperature, top_k)?;
1231        Ok(next)
1232    }
1233
1234    /// Temperature-scaled top-k categorical sampling.
1235    ///
1236    /// Divides logits by `temperature`, masks all but the top-k entries,
1237    /// applies softmax over those k entries, then draws a sample using a
1238    /// xorshift64 PRNG embedded in `self.rng_state`.
1239    fn sample_with_temperature(
1240        &mut self,
1241        logits: &GpuBuffer<f32>,
1242        vocab_size: usize,
1243        temperature: f32,
1244        top_k: u32,
1245    ) -> TptrResult<u32> {
1246        let mut raw = vec![0.0f32; vocab_size];
1247        let n = raw.len().min(logits.num_elements());
1248        let _ = logits.copy_to_host(&mut raw[..n]);
1249
1250        // Apply temperature.
1251        for x in raw.iter_mut() {
1252            *x /= temperature;
1253        }
1254
1255        // Find the top-k indices by partial sort (selection).
1256        let k = (top_k as usize).min(vocab_size).max(1);
1257        let mut indexed: Vec<(usize, f32)> = raw.iter().cloned().enumerate().collect();
1258        indexed.sort_unstable_by(|(_, a), (_, b)| {
1259            b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
1260        });
1261        indexed.truncate(k);
1262
1263        // Numerically-stable softmax over top-k.
1264        let max_val = indexed
1265            .iter()
1266            .map(|(_, v)| *v)
1267            .fold(f32::NEG_INFINITY, f32::max);
1268        let mut expd: Vec<f32> = indexed.iter().map(|(_, v)| (*v - max_val).exp()).collect();
1269        let sum: f32 = expd.iter().sum();
1270        if sum > 0.0 {
1271            for e in expd.iter_mut() {
1272                *e /= sum;
1273            }
1274        }
1275
1276        // Categorical sample via xorshift64.
1277        let u = xorshift64(&mut self.rng_state);
1278        let mut cumsum = 0.0f32;
1279        for (i, p) in expd.iter().enumerate() {
1280            cumsum += p;
1281            if u <= cumsum || i + 1 == k {
1282                return Ok(indexed[i].0 as u32);
1283            }
1284        }
1285        Ok(indexed[0].0 as u32)
1286    }
1287}
1288
1289// ---------------------------------------------------------------------------
1290// Helper functions
1291// ---------------------------------------------------------------------------
1292
1293/// Wrap a single token ID as a `GpuBuffer<i32>` of shape `[1]`.
1294fn token_to_index_buffer(token: u32) -> TptrResult<GpuBuffer<i32>> {
1295    let mut buf = GpuBuffer::<i32>::new(Shape::new(&[1]), DType::I32, BufferFlags::STORAGE)
1296        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1297    buf.copy_from_host(&[token as i32])
1298        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1299    Ok(buf)
1300}
1301
1302/// Re-interpret a `GpuBuffer` as `[rows, cols]` — creates a new buffer with
1303/// the same data and the target shape.
1304fn reshape_to_2d(buf: GpuBuffer<f32>, rows: usize, cols: usize) -> TptrResult<GpuBuffer<f32>> {
1305    let n = rows * cols;
1306    let mut src = vec![0.0f32; buf.num_elements().max(1)];
1307    let copy_len = src.len().min(buf.num_elements());
1308    let _ = buf.copy_to_host(&mut src[..copy_len]);
1309    let mut out = GpuBuffer::<f32>::new(Shape::dim2(rows, cols), DType::F32, BufferFlags::STORAGE)
1310        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1311    let mut dst = vec![0.0f32; n];
1312    let fill_len = dst.len().min(src.len());
1313    dst[..fill_len].copy_from_slice(&src[..fill_len]);
1314    out.copy_from_host(&dst)
1315        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1316    Ok(out)
1317}
1318
1319/// Infer the output dimension of an attention result buffer.
1320fn attn_out_dim(buf: &GpuBuffer<f32>) -> usize {
1321    buf.dim(1).unwrap_or(buf.num_elements())
1322}
1323
1324/// Quantize a float weight matrix and run QuantGemmKernel.
1325///
1326/// Computes `activation @ weight` by packing `weight^T` as `i8` with per-group
1327/// scales and dispatching through `QuantGemmKernel`.  The result has shape
1328/// `[1, out_dim]`, identical to the f32 GEMM path.
1329///
1330/// This establishes the full quantized GEMM code path. When real GGUF tensor
1331/// loading lands, weights should be stored pre-packed instead of quantized here.
1332fn quant_layer_gemm(
1333    quant_k: &QuantGemmKernel,
1334    activation: &GpuBuffer<f32>,
1335    weight: &GpuBuffer<f32>,
1336    bits: u8,
1337) -> TptrResult<GpuBuffer<f32>> {
1338    let in_dim = weight.dim(0).unwrap_or(0);
1339    let out_dim = weight.dim(1).unwrap_or(0);
1340    if in_dim == 0 || out_dim == 0 {
1341        return Err(TptrError::new(
1342            ErrorCode::ArgumentMismatch,
1343            "quant_layer_gemm: weight has zero dimension",
1344        ));
1345    }
1346
1347    // Read weight [in_dim, out_dim] and activation [1, in_dim].
1348    let mut w = vec![0.0f32; in_dim * out_dim];
1349    let wn = w.len().min(weight.num_elements());
1350    let _ = weight.copy_to_host(&mut w[..wn]);
1351
1352    let mut act = vec![0.0f32; in_dim];
1353    let an = act.len().min(activation.num_elements());
1354    let _ = activation.copy_to_host(&mut act[..an]);
1355
1356    // Transpose weight to W^T [out_dim, in_dim] for QuantGemmKernel's row-major layout.
1357    let mut wt = vec![0.0f32; out_dim * in_dim];
1358    for r in 0..in_dim {
1359        for c in 0..out_dim {
1360            wt[c * in_dim + r] = w[r * out_dim + c];
1361        }
1362    }
1363
1364    // Compute per-group scales for W^T rows (= W columns = output neurons).
1365    let bits_u = bits.max(1) as usize;
1366    let pack_factor = (8usize / bits_u).max(1);
1367    let packed_k = in_dim.div_ceil(pack_factor);
1368    let group_size = tpt_gpu_primitives::DEFAULT_GROUP_SIZE as usize;
1369    let groups_per_row = in_dim.div_ceil(group_size);
1370    let max_int = ((1u32 << bits) - 1) as f32 / 2.0;
1371
1372    let mut scales = vec![1.0f32; out_dim * groups_per_row];
1373    for row in 0..out_dim {
1374        for g in 0..groups_per_row {
1375            let start = row * in_dim + g * group_size;
1376            let end = (start + group_size).min(row * in_dim + in_dim);
1377            let max_abs = wt[start..end]
1378                .iter()
1379                .map(|x| x.abs())
1380                .fold(0.0f32, f32::max);
1381            scales[row * groups_per_row + g] = if max_abs > 0.0 {
1382                max_abs / max_int
1383            } else {
1384                1.0
1385            };
1386        }
1387    }
1388
1389    // Pack W^T rows into i8.
1390    let mut packed = vec![0i8; out_dim * packed_k];
1391    for row in 0..out_dim {
1392        for col in 0..in_dim {
1393            let g = col / group_size;
1394            let scale = scales[row * groups_per_row + g];
1395            let val = (wt[row * in_dim + col] / scale)
1396                .round()
1397                .clamp(-max_int, max_int) as i8;
1398            let packed_col = col / pack_factor;
1399            let shift = (col % pack_factor) * bits_u;
1400            packed[row * packed_k + packed_col] |= ((val as u8) << shift) as i8;
1401        }
1402    }
1403
1404    // Build GpuBuffers.
1405    let mut a_buf = GpuBuffer::<i8>::new(
1406        Shape::dim2(out_dim, packed_k),
1407        DType::I8,
1408        BufferFlags::STORAGE,
1409    )
1410    .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1411    a_buf
1412        .copy_from_host(&packed)
1413        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1414
1415    // b_f32 = activation^T [in_dim, 1].
1416    let mut b_buf = GpuBuffer::<f32>::new(Shape::dim2(in_dim, 1), DType::F32, BufferFlags::STORAGE)
1417        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1418    b_buf
1419        .copy_from_host(&act)
1420        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1421
1422    let mut sc_buf = GpuBuffer::<f32>::new(
1423        Shape::dim2(out_dim, groups_per_row),
1424        DType::F32,
1425        BufferFlags::STORAGE,
1426    )
1427    .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1428    sc_buf
1429        .copy_from_host(&scales)
1430        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1431
1432    let zp_buf = GpuBuffer::<i8>::new(
1433        Shape::dim2(out_dim, groups_per_row),
1434        DType::I8,
1435        BufferFlags::STORAGE,
1436    )
1437    .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1438
1439    // dequant(W^T) @ act^T → [out_dim, 1]; reshape to [1, out_dim].
1440    let result = quant_k
1441        .execute(&a_buf, &b_buf, &sc_buf, &zp_buf)
1442        .map_err(|e| TptrError::new(ErrorCode::LaunchFailure, e.to_string()))?;
1443
1444    // Transpose [out_dim, 1] → [1, out_dim].
1445    let mut out_raw = vec![0.0f32; out_dim];
1446    let rn = out_raw.len().min(result.num_elements());
1447    let _ = result.copy_to_host(&mut out_raw[..rn]);
1448    vec_to_buf(out_raw, 1, out_dim)
1449}
1450
1451/// Expand GQA/MQA KV head layout to match the full query-head count.
1452///
1453/// `src` is a flat slice of `[seq_len × kv_row]` floats where `kv_row =
1454/// num_kv_heads × head_dim`.  Returns a `Vec` of `[seq_len × qk_dim]` floats
1455/// where `qk_dim = num_heads × head_dim`, repeating KV heads cyclically to
1456/// fill all query heads (standard GQA broadcasting).
1457fn expand_kv_heads(src: &[f32], seq_len: usize, kv_row: usize, qk_dim: usize) -> Vec<f32> {
1458    if seq_len == 0 || qk_dim == 0 {
1459        return vec![];
1460    }
1461    if kv_row >= qk_dim {
1462        // MHA or already full width — just copy / pad each row.
1463        let mut out = vec![0.0f32; seq_len * qk_dim];
1464        for t in 0..seq_len {
1465            let src_start = t * kv_row;
1466            let dst_start = t * qk_dim;
1467            let copy = qk_dim.min(kv_row).min(src.len().saturating_sub(src_start));
1468            if copy > 0 {
1469                out[dst_start..dst_start + copy].copy_from_slice(&src[src_start..src_start + copy]);
1470            }
1471        }
1472        return out;
1473    }
1474    // GQA: broadcast kv_row → qk_dim by cycling.
1475    let mut out = vec![0.0f32; seq_len * qk_dim];
1476    for t in 0..seq_len {
1477        let src_start = t * kv_row;
1478        let dst_start = t * qk_dim;
1479        for i in 0..qk_dim {
1480            let src_i = src_start + (i % kv_row);
1481            if src_i < src.len() {
1482                out[dst_start + i] = src[src_i];
1483            }
1484        }
1485    }
1486    out
1487}
1488
1489/// Dequantize a packed tensor block (matching the `quantize_tensor` format used
1490/// by `tpt-gpu-model-optimizer`) back to `f32`.
1491///
1492/// For `bits >= 16` the weight bytes are interpreted directly as little-endian
1493/// `f32` values (no quantization was applied during storage).  For sub-byte
1494/// depths the values are extracted LSB-first, then dequantized:
1495/// `result[i] = (q + zero_point) * scale`
1496/// where `q` is the per-element quantized integer and scale/zp are per-group.
1497fn tptf_dequantize(
1498    quantized: &[u8],
1499    scales: &[f32],
1500    zero_points: &[i8],
1501    bits: u8,
1502    group_size: usize,
1503    num_elements: usize,
1504) -> Vec<f32> {
1505    if bits >= 16 {
1506        // Raw f32 LE bytes (bits == 16 → no quantization path in quantize_tensor).
1507        let float_count = num_elements.min(quantized.len() / 4);
1508        let mut result = vec![0.0f32; num_elements];
1509        for i in 0..float_count {
1510            let b = &quantized[i * 4..(i + 1) * 4];
1511            result[i] = f32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1512        }
1513        return result;
1514    }
1515
1516    let pack_factor = (8usize / bits as usize).max(1);
1517    let mask = (1u16 << bits) - 1;
1518    let mut result = vec![0.0f32; num_elements];
1519
1520    for (elem_idx, out) in result.iter_mut().enumerate() {
1521        let packed_byte_idx = elem_idx / pack_factor;
1522        let bit_offset = (elem_idx % pack_factor) * bits as usize;
1523
1524        if packed_byte_idx < quantized.len() {
1525            let byte = quantized[packed_byte_idx] as u16;
1526            let q = (byte >> bit_offset) & mask;
1527
1528            let group_idx = elem_idx / group_size;
1529            let scale = scales.get(group_idx).copied().unwrap_or(1.0);
1530            let zp = zero_points.get(group_idx).copied().unwrap_or(0) as f32;
1531
1532            *out = (q as f32 + zp) * scale;
1533        }
1534    }
1535
1536    result
1537}
1538
1539/// Host-side token sampling from raw logit scores.
1540///
1541/// Deterministic (greedy) when `temperature == 0.0` or `top_k <= 1` — returns
1542/// the argmax index.  Otherwise applies temperature scaling, selects the top-k
1543/// candidates, runs softmax over those k values, and draws a categorical sample
1544/// using an internal xorshift64 PRNG (no external crate required).
1545pub fn execute_sampling(logits: &[f32], temperature: f32, top_k: usize) -> usize {
1546    if logits.is_empty() {
1547        return 0;
1548    }
1549
1550    // Greedy (deterministic) fast path.
1551    if temperature == 0.0 || top_k <= 1 {
1552        return logits
1553            .iter()
1554            .enumerate()
1555            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
1556            .map(|(i, _)| i)
1557            .unwrap_or(0);
1558    }
1559
1560    // Temperature scaling.
1561    let mut scaled: Vec<(usize, f32)> = logits
1562        .iter()
1563        .enumerate()
1564        .map(|(i, &v)| (i, v / temperature))
1565        .collect();
1566
1567    // Partial sort: keep only the top-k entries (descending by score).
1568    let k = top_k.min(logits.len()).max(1);
1569    scaled.sort_unstable_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
1570    scaled.truncate(k);
1571
1572    // Numerically-stable softmax over the top-k scores.
1573    let max_val = scaled
1574        .iter()
1575        .map(|(_, v)| *v)
1576        .fold(f32::NEG_INFINITY, f32::max);
1577    let mut expd: Vec<f32> = scaled.iter().map(|(_, v)| (*v - max_val).exp()).collect();
1578    let sum: f32 = expd.iter().sum();
1579    if sum > 0.0 {
1580        for e in expd.iter_mut() {
1581            *e /= sum;
1582        }
1583    }
1584
1585    // Categorical sample via a module-level xorshift64 PRNG (no external dep).
1586    use std::sync::atomic::{AtomicU64, Ordering};
1587    static PRNG: AtomicU64 = AtomicU64::new(0x853c49e6748fea9b);
1588    let u = {
1589        let mut x = PRNG.load(Ordering::Relaxed);
1590        x ^= x << 13;
1591        x ^= x >> 7;
1592        x ^= x << 17;
1593        PRNG.store(x, Ordering::Relaxed);
1594        (x as f64 / u64::MAX as f64) as f32
1595    };
1596
1597    let mut cumsum = 0.0f32;
1598    for (i, p) in expd.iter().enumerate() {
1599        cumsum += p;
1600        if u <= cumsum || i + 1 == k {
1601            return scaled[i].0;
1602        }
1603    }
1604    scaled[0].0
1605}
1606
1607/// xorshift64 PRNG — returns a uniform float in (0, 1].
1608fn xorshift64(state: &mut u64) -> f32 {
1609    let mut x = *state;
1610    x ^= x << 13;
1611    x ^= x >> 7;
1612    x ^= x << 17;
1613    *state = x;
1614    // Map to (0, 1] by dividing by 2^64.
1615    (x as f64 / u64::MAX as f64) as f32
1616}
1617
1618/// Return a new `Vec<f32>` of exactly `target_len` elements from `src`,
1619/// padding with zeros if `src` is shorter.
1620#[allow(dead_code)]
1621fn pad_to_len(src: &[f32], target_len: usize) -> Vec<f32> {
1622    let mut out = vec![0.0f32; target_len];
1623    let copy = src.len().min(target_len);
1624    out[..copy].copy_from_slice(&src[..copy]);
1625    out
1626}
1627
1628/// Copy GpuBuffer contents to a `Vec<f32>`.
1629fn buf_to_vec(buf: &GpuBuffer<f32>) -> Vec<f32> {
1630    let n = buf.num_elements();
1631    let mut v = vec![0.0f32; n];
1632    let _ = buf.copy_to_host(&mut v);
1633    v
1634}
1635
1636/// Create a `GpuBuffer<f32>` of shape `[rows, cols]` from a `Vec<f32>`.
1637fn vec_to_buf(data: Vec<f32>, rows: usize, cols: usize) -> TptrResult<GpuBuffer<f32>> {
1638    let mut buf = GpuBuffer::<f32>::new(Shape::dim2(rows, cols), DType::F32, BufferFlags::STORAGE)
1639        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1640    let n = (rows * cols).min(data.len());
1641    let mut padded = vec![0.0f32; rows * cols];
1642    padded[..n].copy_from_slice(&data[..n]);
1643    buf.copy_from_host(&padded)
1644        .map_err(|e| TptrError::new(ErrorCode::AllocationFailure, e.to_string()))?;
1645    Ok(buf)
1646}
1647
1648/// Elementwise addition of two `[1, dim]` buffers, host-side.
1649fn elementwise_add_2d(
1650    a: &GpuBuffer<f32>,
1651    b: &GpuBuffer<f32>,
1652    dim: usize,
1653) -> TptrResult<GpuBuffer<f32>> {
1654    let mut va = vec![0.0f32; dim];
1655    let mut vb = vec![0.0f32; dim];
1656    let _ = a.copy_to_host(&mut va);
1657    let bn = vb.len().min(b.num_elements());
1658    let _ = b.copy_to_host(&mut vb[..bn]);
1659    for (x, y) in va.iter_mut().zip(vb.iter()) {
1660        *x += y;
1661    }
1662    vec_to_buf(va, 1, dim)
1663}
1664
1665/// SwiGLU activation: result[i] = silu(gate[i]) * up[i].
1666/// Both `gate` and `up` are `[1, ffn_dim]` buffers.
1667fn swiglu_host(
1668    gate: &GpuBuffer<f32>,
1669    up: &GpuBuffer<f32>,
1670    ffn_dim: usize,
1671) -> TptrResult<GpuBuffer<f32>> {
1672    let mut g = vec![0.0f32; ffn_dim];
1673    let mut u = vec![0.0f32; ffn_dim];
1674    let gn = g.len().min(gate.num_elements());
1675    let _ = gate.copy_to_host(&mut g[..gn]);
1676    let un = u.len().min(up.num_elements());
1677    let _ = up.copy_to_host(&mut u[..un]);
1678    for (gi, ui) in g.iter_mut().zip(u.iter()) {
1679        let silu = *gi / (1.0 + (-*gi).exp()); // silu(x) = x * sigmoid(x)
1680        *gi = silu * ui;
1681    }
1682    vec_to_buf(g, 1, ffn_dim)
1683}
1684
1685/// Tied lm_head: compute logits as hidden @ embed_table^T (host-side).
1686/// `hidden` is [1, hidden_dim]; `embed_table` is [vocab_size, hidden_dim].
1687/// Result is [1, vocab_size].
1688fn tied_lm_head_host(
1689    hidden: &GpuBuffer<f32>,
1690    embed_table: &GpuBuffer<f32>,
1691    hidden_dim: usize,
1692    vocab_size: usize,
1693) -> TptrResult<GpuBuffer<f32>> {
1694    let mut h = vec![0.0f32; hidden_dim];
1695    let mut e = vec![0.0f32; vocab_size * hidden_dim];
1696    let hn = h.len().min(hidden.num_elements());
1697    let _ = hidden.copy_to_host(&mut h[..hn]);
1698    let en = e.len().min(embed_table.num_elements());
1699    let _ = embed_table.copy_to_host(&mut e[..en]);
1700
1701    let mut logits = vec![0.0f32; vocab_size];
1702    for v in 0..vocab_size {
1703        let row = &e[v * hidden_dim..(v + 1) * hidden_dim];
1704        logits[v] = h.iter().zip(row.iter()).map(|(a, b)| a * b).sum();
1705    }
1706    vec_to_buf(logits, 1, vocab_size)
1707}
1708
1709/// Greedy sampling: returns the index of the maximum probability.
1710fn sample_argmax(probs: &GpuBuffer<f32>, vocab_size: usize) -> u32 {
1711    let mut p = vec![0.0f32; vocab_size];
1712    let pn = p.len().min(probs.num_elements());
1713    let _ = probs.copy_to_host(&mut p[..pn]);
1714    p.iter()
1715        .enumerate()
1716        .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
1717        .map(|(i, _)| i as u32)
1718        .unwrap_or(0)
1719}
1720
1721// ---------------------------------------------------------------------------
1722// Tests
1723// ---------------------------------------------------------------------------
1724
1725#[cfg(test)]
1726mod tests {
1727    use super::*;
1728    use std::fs;
1729
1730    /// Write a minimal valid GGUF v2 file with the given KV metadata.
1731    fn write_gguf(path: &Path, kv: &[(&str, GgufKvVal)]) {
1732        let mut buf: Vec<u8> = Vec::new();
1733        buf.extend_from_slice(b"GGUF"); // magic
1734        buf.extend_from_slice(&2u32.to_le_bytes()); // version 2
1735        buf.extend_from_slice(&0u64.to_le_bytes()); // tensor_count
1736        buf.extend_from_slice(&(kv.len() as u64).to_le_bytes()); // kv_count
1737        for (key, val) in kv {
1738            let kb = key.as_bytes();
1739            buf.extend_from_slice(&(kb.len() as u64).to_le_bytes());
1740            buf.extend_from_slice(kb);
1741            match val {
1742                GgufKvVal::Str(s) => {
1743                    buf.extend_from_slice(&8u32.to_le_bytes()); // STRING
1744                    let sb = s.as_bytes();
1745                    buf.extend_from_slice(&(sb.len() as u64).to_le_bytes());
1746                    buf.extend_from_slice(sb);
1747                }
1748                GgufKvVal::U32(v) => {
1749                    buf.extend_from_slice(&4u32.to_le_bytes()); // UINT32
1750                    buf.extend_from_slice(&v.to_le_bytes());
1751                }
1752            }
1753        }
1754        fs::write(path, &buf).unwrap();
1755    }
1756
1757    enum GgufKvVal {
1758        Str(String),
1759        U32(u32),
1760    }
1761
1762    fn tmp(name: &str) -> PathBuf {
1763        let mut p = std::env::temp_dir();
1764        p.push(format!("tptr_infer_{}_{}", std::process::id(), name));
1765        p
1766    }
1767
1768    // ----- GGUF parser tests -----
1769
1770    // ----- TPTF parser tests -----
1771
1772    fn write_tptf(path: &Path, arch: &str, num_layers: u32, per_layer_bits: &[u8]) {
1773        let mut buf = vec![0u8; 512];
1774        buf[0..4].copy_from_slice(b"TPTF");
1775        buf[4..8].copy_from_slice(&1u32.to_le_bytes()); // version
1776        buf[8..12].copy_from_slice(&0u32.to_le_bytes()); // flags
1777        let ab = arch.as_bytes();
1778        buf[12] = ab.len().min(63) as u8;
1779        buf[13..13 + ab.len().min(63)].copy_from_slice(&ab[..ab.len().min(63)]);
1780        buf[76..80].copy_from_slice(&64u32.to_le_bytes()); // context_len
1781        buf[80..84].copy_from_slice(&32000u32.to_le_bytes()); // vocab_size
1782        buf[84..88].copy_from_slice(&64u32.to_le_bytes()); // hidden_dim
1783        buf[88..92].copy_from_slice(&4u32.to_le_bytes()); // num_heads
1784        buf[92..96].copy_from_slice(&2u32.to_le_bytes()); // num_kv_heads
1785        buf[96..100].copy_from_slice(&128u32.to_le_bytes()); // ffn_dim
1786        buf[100..104].copy_from_slice(&num_layers.to_le_bytes());
1787        for (i, &b) in per_layer_bits.iter().enumerate().take(128) {
1788            buf[104 + i] = b;
1789        }
1790        fs::write(path, &buf).unwrap();
1791    }
1792
1793    #[test]
1794    fn parse_tptf_header_basic() {
1795        let p = tmp("basic.tptf");
1796        write_tptf(&p, "llama3", 4, &[4, 4, 6, 8]);
1797        let info = parse_tptf_header(&p).unwrap();
1798        assert_eq!(info.arch, "llama3");
1799        assert_eq!(info.num_layers, 4);
1800        assert_eq!(info.per_layer_bits, vec![4, 4, 6, 8]);
1801        let _ = fs::remove_file(&p);
1802    }
1803
1804    #[test]
1805    fn detect_format_tptf() {
1806        let p = tmp("detect.tptf");
1807        write_tptf(&p, "llama3", 2, &[]);
1808        assert_eq!(detect_format(&p), ModelFormat::Tptf);
1809        let _ = fs::remove_file(&p);
1810    }
1811
1812    #[test]
1813    fn load_tptf_via_engine() {
1814        let p = tmp("engine.tptf");
1815        write_tptf(&p, "llama3", 2, &[4, 4]);
1816        let engine = GpuInferenceEngine::load(&p).unwrap();
1817        assert_eq!(engine.model_info().arch, "llama3");
1818        assert_eq!(engine.model_info().per_layer_bits, vec![4, 4]);
1819        let _ = fs::remove_file(&p);
1820    }
1821
1822    #[test]
1823    fn parse_bad_magic_errors() {
1824        let p = tmp("bad_magic.gguf");
1825        fs::write(
1826            &p,
1827            b"NOTG\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
1828        )
1829        .unwrap();
1830        let err = parse_gguf_header(&p).unwrap_err();
1831        assert_eq!(err.code, ErrorCode::InvalidKernel);
1832        let _ = fs::remove_file(&p);
1833    }
1834
1835    #[test]
1836    fn parse_missing_file_errors() {
1837        let err = parse_gguf_header(Path::new("/nonexistent/model.gguf")).unwrap_err();
1838        assert_eq!(err.code, ErrorCode::InvalidKernel);
1839    }
1840
1841    #[test]
1842    fn parse_extracts_arch_and_dims() {
1843        let p = tmp("parse_arch.gguf");
1844        write_gguf(
1845            &p,
1846            &[
1847                ("general.architecture", GgufKvVal::Str("llama3".into())),
1848                ("llm.context_length", GgufKvVal::U32(8192)),
1849                ("llm.embedding_length", GgufKvVal::U32(4096)),
1850                ("llm.attention.head_count", GgufKvVal::U32(32)),
1851                ("llm.attention.head_count_kv", GgufKvVal::U32(8)),
1852                ("llm.feed_forward_length", GgufKvVal::U32(11008)),
1853                ("llm.block_count", GgufKvVal::U32(32)),
1854            ],
1855        );
1856        let info = parse_gguf_header(&p).unwrap();
1857        assert_eq!(info.arch, "llama3");
1858        assert_eq!(info.context_len, 8192);
1859        assert_eq!(info.hidden_dim, 4096);
1860        assert_eq!(info.num_heads, 32);
1861        assert_eq!(info.num_kv_heads, 8);
1862        assert_eq!(info.num_layers, 32);
1863        let _ = fs::remove_file(&p);
1864    }
1865
1866    #[test]
1867    fn parse_qwen2_arch() {
1868        let p = tmp("qwen2.gguf");
1869        write_gguf(
1870            &p,
1871            &[
1872                ("general.architecture", GgufKvVal::Str("qwen2".into())),
1873                ("llm.context_length", GgufKvVal::U32(32768)),
1874                ("llm.embedding_length", GgufKvVal::U32(3584)),
1875                ("llm.attention.head_count", GgufKvVal::U32(28)),
1876                ("llm.attention.head_count_kv", GgufKvVal::U32(4)),
1877                ("llm.feed_forward_length", GgufKvVal::U32(18944)),
1878                ("llm.block_count", GgufKvVal::U32(28)),
1879            ],
1880        );
1881        let info = parse_gguf_header(&p).unwrap();
1882        assert_eq!(info.arch, "qwen2");
1883        assert_eq!(info.hidden_dim, 3584);
1884        assert_eq!(info.num_kv_heads, 4);
1885        let _ = fs::remove_file(&p);
1886    }
1887
1888    // ----- Engine load + infer tests -----
1889
1890    fn write_minimal_gguf(path: &Path, arch: &str) {
1891        write_gguf(
1892            path,
1893            &[
1894                ("general.architecture", GgufKvVal::Str(arch.into())),
1895                ("llm.context_length", GgufKvVal::U32(64)),
1896                ("llm.embedding_length", GgufKvVal::U32(64)),
1897                ("llm.attention.head_count", GgufKvVal::U32(4)),
1898                ("llm.attention.head_count_kv", GgufKvVal::U32(2)),
1899                ("llm.feed_forward_length", GgufKvVal::U32(128)),
1900                ("llm.block_count", GgufKvVal::U32(2)),
1901            ],
1902        );
1903    }
1904
1905    #[test]
1906    fn load_llama3_succeeds() {
1907        let p = tmp("llama3_load.gguf");
1908        write_minimal_gguf(&p, "llama3");
1909        let engine = GpuInferenceEngine::load(&p).unwrap();
1910        assert_eq!(engine.model_info().arch, "llama3");
1911        let _ = fs::remove_file(&p);
1912    }
1913
1914    #[test]
1915    fn load_qwen2_succeeds() {
1916        let p = tmp("qwen2_load.gguf");
1917        write_minimal_gguf(&p, "qwen2");
1918        let engine = GpuInferenceEngine::load(&p).unwrap();
1919        assert_eq!(engine.model_info().arch, "qwen2");
1920        let _ = fs::remove_file(&p);
1921    }
1922
1923    #[test]
1924    fn load_unknown_arch_errors() {
1925        let p = tmp("gpt2_load.gguf");
1926        write_minimal_gguf(&p, "gpt2");
1927        let err = GpuInferenceEngine::load(&p).unwrap_err();
1928        assert_eq!(err.code, ErrorCode::UnsupportedFeature);
1929        let _ = fs::remove_file(&p);
1930    }
1931
1932    #[test]
1933    fn infer_produces_max_new_tokens() {
1934        let p = tmp("infer_count.gguf");
1935        write_minimal_gguf(&p, "llama3");
1936        let mut engine = GpuInferenceEngine::load(&p).unwrap();
1937        let mut generated = Vec::new();
1938        engine
1939            .infer(&[1, 2, 3], 4, |tok| generated.push(tok))
1940            .unwrap();
1941        assert_eq!(generated.len(), 4);
1942        let _ = fs::remove_file(&p);
1943    }
1944
1945    #[test]
1946    fn infer_empty_tokens_errors() {
1947        let p = tmp("infer_empty.gguf");
1948        write_minimal_gguf(&p, "llama3");
1949        let mut engine = GpuInferenceEngine::load(&p).unwrap();
1950        let err = engine.infer(&[], 4, |_| {}).unwrap_err();
1951        assert_eq!(err.code, ErrorCode::ArgumentMismatch);
1952        let _ = fs::remove_file(&p);
1953    }
1954
1955    #[test]
1956    fn cancel_stops_generation() {
1957        let p = tmp("cancel_gen.gguf");
1958        write_minimal_gguf(&p, "llama3");
1959        let mut engine = GpuInferenceEngine::load(&p).unwrap();
1960        engine.cancel();
1961        let mut generated = Vec::new();
1962        engine
1963            .infer(&[1, 2, 3], 100, |tok| generated.push(tok))
1964            .unwrap();
1965        assert!(
1966            generated.is_empty(),
1967            "cancel before infer should produce no tokens"
1968        );
1969        let _ = fs::remove_file(&p);
1970    }
1971
1972    #[test]
1973    fn infer_gemma2_succeeds() {
1974        let p = tmp("gemma2_infer.gguf");
1975        write_minimal_gguf(&p, "gemma2");
1976        let mut engine = GpuInferenceEngine::load(&p).unwrap();
1977        let mut generated = Vec::new();
1978        engine.infer(&[1], 2, |tok| generated.push(tok)).unwrap();
1979        assert_eq!(generated.len(), 2);
1980        let _ = fs::remove_file(&p);
1981    }
1982
1983    // ----- TPTF tensor-loading roundtrip -----
1984
1985    /// Write a TPTF block binary (matching TensorBlock::write_to from tpt-gpu-model-optimizer).
1986    #[allow(clippy::too_many_arguments)]
1987    fn append_tptf_tensor_block(
1988        data: &mut Vec<u8>,
1989        layer_idx: u32,
1990        name: &str,
1991        bits: u8,
1992        group_size: u32,
1993        rows: u32,
1994        cols: u32,
1995        weight_bytes: &[u8],
1996    ) {
1997        data.extend_from_slice(&layer_idx.to_le_bytes());
1998        let nb = name.as_bytes();
1999        data.push(nb.len().min(31) as u8);
2000        data.extend_from_slice(&nb[..nb.len().min(31)]);
2001        data.push(bits);
2002        data.extend_from_slice(&group_size.to_le_bytes());
2003        data.extend_from_slice(&rows.to_le_bytes());
2004        data.extend_from_slice(&cols.to_le_bytes());
2005        // weights
2006        data.extend_from_slice(&(weight_bytes.len() as u32).to_le_bytes());
2007        data.extend_from_slice(weight_bytes);
2008        // scales: 1 group, scale = 1.0
2009        data.extend_from_slice(&1u32.to_le_bytes());
2010        data.extend_from_slice(&1.0f32.to_le_bytes());
2011        // zero_points: 1 group, zp = 0
2012        data.extend_from_slice(&1u32.to_le_bytes());
2013        data.push(0u8);
2014        // pad absolute position to next 128-byte boundary
2015        while !data.len().is_multiple_of(128) {
2016            data.push(0u8);
2017        }
2018    }
2019
2020    /// Build a minimal TPTF file containing 2 f32 tensor blocks of known values
2021    /// (bits=16 → raw f32 bytes, no quantization).
2022    fn write_tptf_with_tensors(path: &Path) {
2023        // Header: arch=llama3, hidden=4, ffn=8, heads=2, kv=1, layers=1, vocab=8
2024        let mut buf = vec![0u8; 512];
2025        buf[0..4].copy_from_slice(b"TPTF");
2026        buf[4..8].copy_from_slice(&1u32.to_le_bytes()); // version
2027        buf[8..12].copy_from_slice(&0u32.to_le_bytes()); // flags
2028        let ab = b"llama3";
2029        buf[12] = ab.len() as u8;
2030        buf[13..13 + ab.len()].copy_from_slice(ab);
2031        buf[76..80].copy_from_slice(&64u32.to_le_bytes()); // context_len
2032        buf[80..84].copy_from_slice(&8u32.to_le_bytes()); // vocab_size
2033        buf[84..88].copy_from_slice(&4u32.to_le_bytes()); // hidden_dim
2034        buf[88..92].copy_from_slice(&2u32.to_le_bytes()); // num_heads
2035        buf[92..96].copy_from_slice(&1u32.to_le_bytes()); // num_kv_heads
2036        buf[96..100].copy_from_slice(&8u32.to_le_bytes()); // ffn_dim
2037        buf[100..104].copy_from_slice(&1u32.to_le_bytes()); // num_layers = 1
2038                                                            // tensor_offset at byte 460 = 512 (start of tensor section)
2039        buf[460..468].copy_from_slice(&512u64.to_le_bytes());
2040        // tokenizer_offset = 0 (no tokenizer → read tensors until EOF)
2041
2042        let mut data = buf;
2043
2044        // Tensor 1: gate_proj for layer 0 — 32 elements (4 × 8), bits=16
2045        let gate_vals: Vec<f32> = (1u32..=32).map(|i| i as f32 * 0.1).collect();
2046        let gate_bytes: Vec<u8> = gate_vals.iter().flat_map(|v| v.to_le_bytes()).collect();
2047        append_tptf_tensor_block(&mut data, 0, "gate_proj", 16, 32, 4, 8, &gate_bytes);
2048
2049        // Tensor 2: up_proj for layer 0 — 32 elements (4 × 8), bits=16
2050        let up_vals: Vec<f32> = (1u32..=32).map(|i| i as f32 * 0.2).collect();
2051        let up_bytes: Vec<u8> = up_vals.iter().flat_map(|v| v.to_le_bytes()).collect();
2052        append_tptf_tensor_block(&mut data, 0, "up_proj", 16, 32, 4, 8, &up_bytes);
2053
2054        fs::write(path, &data).unwrap();
2055    }
2056
2057    #[test]
2058    fn test_load_tptf_roundtrip() {
2059        let p = tmp("tptf_roundtrip.tptf");
2060        write_tptf_with_tensors(&p);
2061
2062        let info = parse_tptf_header(&p).unwrap();
2063        assert_eq!(info.arch, "llama3");
2064        assert_eq!(info.num_layers, 1);
2065        assert_eq!(info.hidden_dim, 4);
2066        assert_eq!(info.ffn_dim, 8);
2067
2068        // llama3 does not tie lm_head
2069        let weights = ModelWeights::load_tptf(&p, &info, /*tied=*/ false).unwrap();
2070
2071        // gate_proj[layer=0] should be (0.1, 0.2, ..., 3.2)
2072        let gate = buf_to_vec(&weights.layers[0].gate_proj);
2073        assert_eq!(gate.len(), 4 * 8, "gate_proj shape mismatch");
2074        for (i, &v) in gate.iter().enumerate() {
2075            let expected = (i + 1) as f32 * 0.1;
2076            assert!(
2077                (v - expected).abs() < 1e-5,
2078                "gate_proj[{}]: expected {:.4}, got {:.4}",
2079                i,
2080                expected,
2081                v
2082            );
2083        }
2084
2085        // up_proj[layer=0] should be (0.2, 0.4, ..., 6.4)
2086        let up = buf_to_vec(&weights.layers[0].up_proj);
2087        assert_eq!(up.len(), 4 * 8, "up_proj shape mismatch");
2088        for (i, &v) in up.iter().enumerate() {
2089            let expected = (i + 1) as f32 * 0.2;
2090            assert!(
2091                (v - expected).abs() < 1e-5,
2092                "up_proj[{}]: expected {:.4}, got {:.4}",
2093                i,
2094                expected,
2095                v
2096            );
2097        }
2098
2099        let _ = fs::remove_file(&p);
2100    }
2101
2102    // ----- execute_sampling tests -----
2103
2104    #[test]
2105    fn test_sampling_temperature_zero_is_argmax() {
2106        // With temperature=0, execute_sampling must return the argmax index
2107        // regardless of the values of the other logits.
2108        let logits = vec![0.1f32, 0.5, 0.9, 0.3, 0.2];
2109        let idx = execute_sampling(&logits, 0.0, 5);
2110        assert_eq!(
2111            idx, 2,
2112            "argmax of [0.1,0.5,0.9,0.3,0.2] should be 2, got {}",
2113            idx
2114        );
2115
2116        // Also verify with a negative max-at-zero case.
2117        let logits2 = vec![10.0f32, 1.0, 2.0];
2118        let idx2 = execute_sampling(&logits2, 0.0, 3);
2119        assert_eq!(idx2, 0, "argmax of [10,1,2] should be 0, got {}", idx2);
2120    }
2121
2122    #[test]
2123    fn test_sampling_topk_one_is_argmax() {
2124        // With top_k=1, only the highest-logit token can be sampled — the result
2125        // is always the argmax regardless of the temperature value.
2126        let logits = vec![0.1f32, 0.8, 0.3, 0.6];
2127        let idx = execute_sampling(&logits, 1.0, 1);
2128        assert_eq!(idx, 1, "top_k=1 should return argmax (1), got {}", idx);
2129
2130        let logits2 = vec![3.0f32, 1.0, 2.0, 5.0, 4.0];
2131        let idx2 = execute_sampling(&logits2, 2.0, 1);
2132        assert_eq!(idx2, 3, "top_k=1 should return argmax (3), got {}", idx2);
2133    }
2134}